sched/numa: Kill the wrong/dead TASK_DEAD check in task_numa_fault()
[deliverable/linux.git] / kernel / sched / core.c
CommitLineData
1da177e4 1/*
391e43da 2 * kernel/sched/core.c
1da177e4
LT
3 *
4 * Kernel scheduler and related syscalls
5 *
6 * Copyright (C) 1991-2002 Linus Torvalds
7 *
8 * 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and
9 * make semaphores SMP safe
10 * 1998-11-19 Implemented schedule_timeout() and related stuff
11 * by Andrea Arcangeli
12 * 2002-01-04 New ultra-scalable O(1) scheduler by Ingo Molnar:
13 * hybrid priority-list and round-robin design with
14 * an array-switch method of distributing timeslices
15 * and per-CPU runqueues. Cleanups and useful suggestions
16 * by Davide Libenzi, preemptible kernel bits by Robert Love.
17 * 2003-09-03 Interactivity tuning by Con Kolivas.
18 * 2004-04-02 Scheduler domains code by Nick Piggin
c31f2e8a
IM
19 * 2007-04-15 Work begun on replacing all interactivity tuning with a
20 * fair scheduling design by Con Kolivas.
21 * 2007-05-05 Load balancing (smp-nice) and other improvements
22 * by Peter Williams
23 * 2007-05-06 Interactivity improvements to CFS by Mike Galbraith
24 * 2007-07-01 Group scheduling enhancements by Srivatsa Vaddagiri
b9131769
IM
25 * 2007-11-29 RT balancing improvements by Steven Rostedt, Gregory Haskins,
26 * Thomas Gleixner, Mike Kravetz
1da177e4
LT
27 */
28
29#include <linux/mm.h>
30#include <linux/module.h>
31#include <linux/nmi.h>
32#include <linux/init.h>
dff06c15 33#include <linux/uaccess.h>
1da177e4 34#include <linux/highmem.h>
1da177e4
LT
35#include <asm/mmu_context.h>
36#include <linux/interrupt.h>
c59ede7b 37#include <linux/capability.h>
1da177e4
LT
38#include <linux/completion.h>
39#include <linux/kernel_stat.h>
9a11b49a 40#include <linux/debug_locks.h>
cdd6c482 41#include <linux/perf_event.h>
1da177e4
LT
42#include <linux/security.h>
43#include <linux/notifier.h>
44#include <linux/profile.h>
7dfb7103 45#include <linux/freezer.h>
198e2f18 46#include <linux/vmalloc.h>
1da177e4
LT
47#include <linux/blkdev.h>
48#include <linux/delay.h>
b488893a 49#include <linux/pid_namespace.h>
1da177e4
LT
50#include <linux/smp.h>
51#include <linux/threads.h>
52#include <linux/timer.h>
53#include <linux/rcupdate.h>
54#include <linux/cpu.h>
55#include <linux/cpuset.h>
56#include <linux/percpu.h>
b5aadf7f 57#include <linux/proc_fs.h>
1da177e4 58#include <linux/seq_file.h>
e692ab53 59#include <linux/sysctl.h>
1da177e4
LT
60#include <linux/syscalls.h>
61#include <linux/times.h>
8f0ab514 62#include <linux/tsacct_kern.h>
c6fd91f0 63#include <linux/kprobes.h>
0ff92245 64#include <linux/delayacct.h>
dff06c15 65#include <linux/unistd.h>
f5ff8422 66#include <linux/pagemap.h>
8f4d37ec 67#include <linux/hrtimer.h>
30914a58 68#include <linux/tick.h>
f00b45c1
PZ
69#include <linux/debugfs.h>
70#include <linux/ctype.h>
6cd8a4bb 71#include <linux/ftrace.h>
5a0e3ad6 72#include <linux/slab.h>
f1c6f1a7 73#include <linux/init_task.h>
40401530 74#include <linux/binfmts.h>
91d1aa43 75#include <linux/context_tracking.h>
52f5684c 76#include <linux/compiler.h>
1da177e4 77
96f951ed 78#include <asm/switch_to.h>
5517d86b 79#include <asm/tlb.h>
838225b4 80#include <asm/irq_regs.h>
db7e527d 81#include <asm/mutex.h>
e6e6685a
GC
82#ifdef CONFIG_PARAVIRT
83#include <asm/paravirt.h>
84#endif
1da177e4 85
029632fb 86#include "sched.h"
ea138446 87#include "../workqueue_internal.h"
29d5e047 88#include "../smpboot.h"
6e0534f2 89
a8d154b0 90#define CREATE_TRACE_POINTS
ad8d75ff 91#include <trace/events/sched.h>
a8d154b0 92
febdbfe8
PZ
93#ifdef smp_mb__before_atomic
94void __smp_mb__before_atomic(void)
95{
96 smp_mb__before_atomic();
97}
98EXPORT_SYMBOL(__smp_mb__before_atomic);
99#endif
100
101#ifdef smp_mb__after_atomic
102void __smp_mb__after_atomic(void)
103{
104 smp_mb__after_atomic();
105}
106EXPORT_SYMBOL(__smp_mb__after_atomic);
107#endif
108
029632fb 109void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period)
d0b27fa7 110{
58088ad0
PT
111 unsigned long delta;
112 ktime_t soft, hard, now;
d0b27fa7 113
58088ad0
PT
114 for (;;) {
115 if (hrtimer_active(period_timer))
116 break;
117
118 now = hrtimer_cb_get_time(period_timer);
119 hrtimer_forward(period_timer, now, period);
d0b27fa7 120
58088ad0
PT
121 soft = hrtimer_get_softexpires(period_timer);
122 hard = hrtimer_get_expires(period_timer);
123 delta = ktime_to_ns(ktime_sub(hard, soft));
124 __hrtimer_start_range_ns(period_timer, soft, delta,
125 HRTIMER_MODE_ABS_PINNED, 0);
126 }
127}
128
029632fb
PZ
129DEFINE_MUTEX(sched_domains_mutex);
130DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
dc61b1d6 131
fe44d621 132static void update_rq_clock_task(struct rq *rq, s64 delta);
305e6835 133
029632fb 134void update_rq_clock(struct rq *rq)
3e51f33f 135{
fe44d621 136 s64 delta;
305e6835 137
61eadef6 138 if (rq->skip_clock_update > 0)
f26f9aff 139 return;
aa483808 140
fe44d621 141 delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;
4036ac15
MG
142 if (delta < 0)
143 return;
fe44d621
PZ
144 rq->clock += delta;
145 update_rq_clock_task(rq, delta);
3e51f33f
PZ
146}
147
bf5c91ba
IM
148/*
149 * Debugging: various feature bits
150 */
f00b45c1 151
f00b45c1
PZ
152#define SCHED_FEAT(name, enabled) \
153 (1UL << __SCHED_FEAT_##name) * enabled |
154
bf5c91ba 155const_debug unsigned int sysctl_sched_features =
391e43da 156#include "features.h"
f00b45c1
PZ
157 0;
158
159#undef SCHED_FEAT
160
161#ifdef CONFIG_SCHED_DEBUG
162#define SCHED_FEAT(name, enabled) \
163 #name ,
164
1292531f 165static const char * const sched_feat_names[] = {
391e43da 166#include "features.h"
f00b45c1
PZ
167};
168
169#undef SCHED_FEAT
170
34f3a814 171static int sched_feat_show(struct seq_file *m, void *v)
f00b45c1 172{
f00b45c1
PZ
173 int i;
174
f8b6d1cc 175 for (i = 0; i < __SCHED_FEAT_NR; i++) {
34f3a814
LZ
176 if (!(sysctl_sched_features & (1UL << i)))
177 seq_puts(m, "NO_");
178 seq_printf(m, "%s ", sched_feat_names[i]);
f00b45c1 179 }
34f3a814 180 seq_puts(m, "\n");
f00b45c1 181
34f3a814 182 return 0;
f00b45c1
PZ
183}
184
f8b6d1cc
PZ
185#ifdef HAVE_JUMP_LABEL
186
c5905afb
IM
187#define jump_label_key__true STATIC_KEY_INIT_TRUE
188#define jump_label_key__false STATIC_KEY_INIT_FALSE
f8b6d1cc
PZ
189
190#define SCHED_FEAT(name, enabled) \
191 jump_label_key__##enabled ,
192
c5905afb 193struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
f8b6d1cc
PZ
194#include "features.h"
195};
196
197#undef SCHED_FEAT
198
199static void sched_feat_disable(int i)
200{
c5905afb
IM
201 if (static_key_enabled(&sched_feat_keys[i]))
202 static_key_slow_dec(&sched_feat_keys[i]);
f8b6d1cc
PZ
203}
204
205static void sched_feat_enable(int i)
206{
c5905afb
IM
207 if (!static_key_enabled(&sched_feat_keys[i]))
208 static_key_slow_inc(&sched_feat_keys[i]);
f8b6d1cc
PZ
209}
210#else
211static void sched_feat_disable(int i) { };
212static void sched_feat_enable(int i) { };
213#endif /* HAVE_JUMP_LABEL */
214
1a687c2e 215static int sched_feat_set(char *cmp)
f00b45c1 216{
f00b45c1 217 int i;
1a687c2e 218 int neg = 0;
f00b45c1 219
524429c3 220 if (strncmp(cmp, "NO_", 3) == 0) {
f00b45c1
PZ
221 neg = 1;
222 cmp += 3;
223 }
224
f8b6d1cc 225 for (i = 0; i < __SCHED_FEAT_NR; i++) {
7740191c 226 if (strcmp(cmp, sched_feat_names[i]) == 0) {
f8b6d1cc 227 if (neg) {
f00b45c1 228 sysctl_sched_features &= ~(1UL << i);
f8b6d1cc
PZ
229 sched_feat_disable(i);
230 } else {
f00b45c1 231 sysctl_sched_features |= (1UL << i);
f8b6d1cc
PZ
232 sched_feat_enable(i);
233 }
f00b45c1
PZ
234 break;
235 }
236 }
237
1a687c2e
MG
238 return i;
239}
240
241static ssize_t
242sched_feat_write(struct file *filp, const char __user *ubuf,
243 size_t cnt, loff_t *ppos)
244{
245 char buf[64];
246 char *cmp;
247 int i;
5cd08fbf 248 struct inode *inode;
1a687c2e
MG
249
250 if (cnt > 63)
251 cnt = 63;
252
253 if (copy_from_user(&buf, ubuf, cnt))
254 return -EFAULT;
255
256 buf[cnt] = 0;
257 cmp = strstrip(buf);
258
5cd08fbf
JB
259 /* Ensure the static_key remains in a consistent state */
260 inode = file_inode(filp);
261 mutex_lock(&inode->i_mutex);
1a687c2e 262 i = sched_feat_set(cmp);
5cd08fbf 263 mutex_unlock(&inode->i_mutex);
f8b6d1cc 264 if (i == __SCHED_FEAT_NR)
f00b45c1
PZ
265 return -EINVAL;
266
42994724 267 *ppos += cnt;
f00b45c1
PZ
268
269 return cnt;
270}
271
34f3a814
LZ
272static int sched_feat_open(struct inode *inode, struct file *filp)
273{
274 return single_open(filp, sched_feat_show, NULL);
275}
276
828c0950 277static const struct file_operations sched_feat_fops = {
34f3a814
LZ
278 .open = sched_feat_open,
279 .write = sched_feat_write,
280 .read = seq_read,
281 .llseek = seq_lseek,
282 .release = single_release,
f00b45c1
PZ
283};
284
285static __init int sched_init_debug(void)
286{
f00b45c1
PZ
287 debugfs_create_file("sched_features", 0644, NULL, NULL,
288 &sched_feat_fops);
289
290 return 0;
291}
292late_initcall(sched_init_debug);
f8b6d1cc 293#endif /* CONFIG_SCHED_DEBUG */
bf5c91ba 294
b82d9fdd
PZ
295/*
296 * Number of tasks to iterate in a single balance run.
297 * Limited because this is done with IRQs disabled.
298 */
299const_debug unsigned int sysctl_sched_nr_migrate = 32;
300
e9e9250b
PZ
301/*
302 * period over which we average the RT time consumption, measured
303 * in ms.
304 *
305 * default: 1s
306 */
307const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC;
308
fa85ae24 309/*
9f0c1e56 310 * period over which we measure -rt task cpu usage in us.
fa85ae24
PZ
311 * default: 1s
312 */
9f0c1e56 313unsigned int sysctl_sched_rt_period = 1000000;
fa85ae24 314
029632fb 315__read_mostly int scheduler_running;
6892b75e 316
9f0c1e56
PZ
317/*
318 * part of the period that we allow rt tasks to run in us.
319 * default: 0.95s
320 */
321int sysctl_sched_rt_runtime = 950000;
fa85ae24 322
0970d299 323/*
0122ec5b 324 * __task_rq_lock - lock the rq @p resides on.
b29739f9 325 */
70b97a7f 326static inline struct rq *__task_rq_lock(struct task_struct *p)
b29739f9
IM
327 __acquires(rq->lock)
328{
0970d299
PZ
329 struct rq *rq;
330
0122ec5b
PZ
331 lockdep_assert_held(&p->pi_lock);
332
3a5c359a 333 for (;;) {
0970d299 334 rq = task_rq(p);
05fa785c 335 raw_spin_lock(&rq->lock);
cca26e80 336 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p)))
3a5c359a 337 return rq;
05fa785c 338 raw_spin_unlock(&rq->lock);
cca26e80
KT
339
340 while (unlikely(task_on_rq_migrating(p)))
341 cpu_relax();
b29739f9 342 }
b29739f9
IM
343}
344
1da177e4 345/*
0122ec5b 346 * task_rq_lock - lock p->pi_lock and lock the rq @p resides on.
1da177e4 347 */
70b97a7f 348static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
0122ec5b 349 __acquires(p->pi_lock)
1da177e4
LT
350 __acquires(rq->lock)
351{
70b97a7f 352 struct rq *rq;
1da177e4 353
3a5c359a 354 for (;;) {
0122ec5b 355 raw_spin_lock_irqsave(&p->pi_lock, *flags);
3a5c359a 356 rq = task_rq(p);
05fa785c 357 raw_spin_lock(&rq->lock);
cca26e80 358 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p)))
3a5c359a 359 return rq;
0122ec5b
PZ
360 raw_spin_unlock(&rq->lock);
361 raw_spin_unlock_irqrestore(&p->pi_lock, *flags);
cca26e80
KT
362
363 while (unlikely(task_on_rq_migrating(p)))
364 cpu_relax();
1da177e4 365 }
1da177e4
LT
366}
367
a9957449 368static void __task_rq_unlock(struct rq *rq)
b29739f9
IM
369 __releases(rq->lock)
370{
05fa785c 371 raw_spin_unlock(&rq->lock);
b29739f9
IM
372}
373
0122ec5b
PZ
374static inline void
375task_rq_unlock(struct rq *rq, struct task_struct *p, unsigned long *flags)
1da177e4 376 __releases(rq->lock)
0122ec5b 377 __releases(p->pi_lock)
1da177e4 378{
0122ec5b
PZ
379 raw_spin_unlock(&rq->lock);
380 raw_spin_unlock_irqrestore(&p->pi_lock, *flags);
1da177e4
LT
381}
382
1da177e4 383/*
cc2a73b5 384 * this_rq_lock - lock this runqueue and disable interrupts.
1da177e4 385 */
a9957449 386static struct rq *this_rq_lock(void)
1da177e4
LT
387 __acquires(rq->lock)
388{
70b97a7f 389 struct rq *rq;
1da177e4
LT
390
391 local_irq_disable();
392 rq = this_rq();
05fa785c 393 raw_spin_lock(&rq->lock);
1da177e4
LT
394
395 return rq;
396}
397
8f4d37ec
PZ
398#ifdef CONFIG_SCHED_HRTICK
399/*
400 * Use HR-timers to deliver accurate preemption points.
8f4d37ec 401 */
8f4d37ec 402
8f4d37ec
PZ
403static void hrtick_clear(struct rq *rq)
404{
405 if (hrtimer_active(&rq->hrtick_timer))
406 hrtimer_cancel(&rq->hrtick_timer);
407}
408
8f4d37ec
PZ
409/*
410 * High-resolution timer tick.
411 * Runs from hardirq context with interrupts disabled.
412 */
413static enum hrtimer_restart hrtick(struct hrtimer *timer)
414{
415 struct rq *rq = container_of(timer, struct rq, hrtick_timer);
416
417 WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
418
05fa785c 419 raw_spin_lock(&rq->lock);
3e51f33f 420 update_rq_clock(rq);
8f4d37ec 421 rq->curr->sched_class->task_tick(rq, rq->curr, 1);
05fa785c 422 raw_spin_unlock(&rq->lock);
8f4d37ec
PZ
423
424 return HRTIMER_NORESTART;
425}
426
95e904c7 427#ifdef CONFIG_SMP
971ee28c
PZ
428
429static int __hrtick_restart(struct rq *rq)
430{
431 struct hrtimer *timer = &rq->hrtick_timer;
432 ktime_t time = hrtimer_get_softexpires(timer);
433
434 return __hrtimer_start_range_ns(timer, time, 0, HRTIMER_MODE_ABS_PINNED, 0);
435}
436
31656519
PZ
437/*
438 * called from hardirq (IPI) context
439 */
440static void __hrtick_start(void *arg)
b328ca18 441{
31656519 442 struct rq *rq = arg;
b328ca18 443
05fa785c 444 raw_spin_lock(&rq->lock);
971ee28c 445 __hrtick_restart(rq);
31656519 446 rq->hrtick_csd_pending = 0;
05fa785c 447 raw_spin_unlock(&rq->lock);
b328ca18
PZ
448}
449
31656519
PZ
450/*
451 * Called to set the hrtick timer state.
452 *
453 * called with rq->lock held and irqs disabled
454 */
029632fb 455void hrtick_start(struct rq *rq, u64 delay)
b328ca18 456{
31656519 457 struct hrtimer *timer = &rq->hrtick_timer;
177ef2a6 458 ktime_t time;
459 s64 delta;
460
461 /*
462 * Don't schedule slices shorter than 10000ns, that just
463 * doesn't make sense and can cause timer DoS.
464 */
465 delta = max_t(s64, delay, 10000LL);
466 time = ktime_add_ns(timer->base->get_time(), delta);
b328ca18 467
cc584b21 468 hrtimer_set_expires(timer, time);
31656519
PZ
469
470 if (rq == this_rq()) {
971ee28c 471 __hrtick_restart(rq);
31656519 472 } else if (!rq->hrtick_csd_pending) {
c46fff2a 473 smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd);
31656519
PZ
474 rq->hrtick_csd_pending = 1;
475 }
b328ca18
PZ
476}
477
478static int
479hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu)
480{
481 int cpu = (int)(long)hcpu;
482
483 switch (action) {
484 case CPU_UP_CANCELED:
485 case CPU_UP_CANCELED_FROZEN:
486 case CPU_DOWN_PREPARE:
487 case CPU_DOWN_PREPARE_FROZEN:
488 case CPU_DEAD:
489 case CPU_DEAD_FROZEN:
31656519 490 hrtick_clear(cpu_rq(cpu));
b328ca18
PZ
491 return NOTIFY_OK;
492 }
493
494 return NOTIFY_DONE;
495}
496
fa748203 497static __init void init_hrtick(void)
b328ca18
PZ
498{
499 hotcpu_notifier(hotplug_hrtick, 0);
500}
31656519
PZ
501#else
502/*
503 * Called to set the hrtick timer state.
504 *
505 * called with rq->lock held and irqs disabled
506 */
029632fb 507void hrtick_start(struct rq *rq, u64 delay)
31656519 508{
7f1e2ca9 509 __hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0,
5c333864 510 HRTIMER_MODE_REL_PINNED, 0);
31656519 511}
b328ca18 512
006c75f1 513static inline void init_hrtick(void)
8f4d37ec 514{
8f4d37ec 515}
31656519 516#endif /* CONFIG_SMP */
8f4d37ec 517
31656519 518static void init_rq_hrtick(struct rq *rq)
8f4d37ec 519{
31656519
PZ
520#ifdef CONFIG_SMP
521 rq->hrtick_csd_pending = 0;
8f4d37ec 522
31656519
PZ
523 rq->hrtick_csd.flags = 0;
524 rq->hrtick_csd.func = __hrtick_start;
525 rq->hrtick_csd.info = rq;
526#endif
8f4d37ec 527
31656519
PZ
528 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
529 rq->hrtick_timer.function = hrtick;
8f4d37ec 530}
006c75f1 531#else /* CONFIG_SCHED_HRTICK */
8f4d37ec
PZ
532static inline void hrtick_clear(struct rq *rq)
533{
534}
535
8f4d37ec
PZ
536static inline void init_rq_hrtick(struct rq *rq)
537{
538}
539
b328ca18
PZ
540static inline void init_hrtick(void)
541{
542}
006c75f1 543#endif /* CONFIG_SCHED_HRTICK */
8f4d37ec 544
fd99f91a
PZ
545/*
546 * cmpxchg based fetch_or, macro so it works for different integer types
547 */
548#define fetch_or(ptr, val) \
549({ typeof(*(ptr)) __old, __val = *(ptr); \
550 for (;;) { \
551 __old = cmpxchg((ptr), __val, __val | (val)); \
552 if (__old == __val) \
553 break; \
554 __val = __old; \
555 } \
556 __old; \
557})
558
e3baac47 559#if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG)
fd99f91a
PZ
560/*
561 * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG,
562 * this avoids any races wrt polling state changes and thereby avoids
563 * spurious IPIs.
564 */
565static bool set_nr_and_not_polling(struct task_struct *p)
566{
567 struct thread_info *ti = task_thread_info(p);
568 return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG);
569}
e3baac47
PZ
570
571/*
572 * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set.
573 *
574 * If this returns true, then the idle task promises to call
575 * sched_ttwu_pending() and reschedule soon.
576 */
577static bool set_nr_if_polling(struct task_struct *p)
578{
579 struct thread_info *ti = task_thread_info(p);
580 typeof(ti->flags) old, val = ACCESS_ONCE(ti->flags);
581
582 for (;;) {
583 if (!(val & _TIF_POLLING_NRFLAG))
584 return false;
585 if (val & _TIF_NEED_RESCHED)
586 return true;
587 old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED);
588 if (old == val)
589 break;
590 val = old;
591 }
592 return true;
593}
594
fd99f91a
PZ
595#else
596static bool set_nr_and_not_polling(struct task_struct *p)
597{
598 set_tsk_need_resched(p);
599 return true;
600}
e3baac47
PZ
601
602#ifdef CONFIG_SMP
603static bool set_nr_if_polling(struct task_struct *p)
604{
605 return false;
606}
607#endif
fd99f91a
PZ
608#endif
609
c24d20db 610/*
8875125e 611 * resched_curr - mark rq's current task 'to be rescheduled now'.
c24d20db
IM
612 *
613 * On UP this means the setting of the need_resched flag, on SMP it
614 * might also involve a cross-CPU call to trigger the scheduler on
615 * the target CPU.
616 */
8875125e 617void resched_curr(struct rq *rq)
c24d20db 618{
8875125e 619 struct task_struct *curr = rq->curr;
c24d20db
IM
620 int cpu;
621
8875125e 622 lockdep_assert_held(&rq->lock);
c24d20db 623
8875125e 624 if (test_tsk_need_resched(curr))
c24d20db
IM
625 return;
626
8875125e 627 cpu = cpu_of(rq);
fd99f91a 628
f27dde8d 629 if (cpu == smp_processor_id()) {
8875125e 630 set_tsk_need_resched(curr);
f27dde8d 631 set_preempt_need_resched();
c24d20db 632 return;
f27dde8d 633 }
c24d20db 634
8875125e 635 if (set_nr_and_not_polling(curr))
c24d20db 636 smp_send_reschedule(cpu);
dfc68f29
AL
637 else
638 trace_sched_wake_idle_without_ipi(cpu);
c24d20db
IM
639}
640
029632fb 641void resched_cpu(int cpu)
c24d20db
IM
642{
643 struct rq *rq = cpu_rq(cpu);
644 unsigned long flags;
645
05fa785c 646 if (!raw_spin_trylock_irqsave(&rq->lock, flags))
c24d20db 647 return;
8875125e 648 resched_curr(rq);
05fa785c 649 raw_spin_unlock_irqrestore(&rq->lock, flags);
c24d20db 650}
06d8308c 651
b021fe3e 652#ifdef CONFIG_SMP
3451d024 653#ifdef CONFIG_NO_HZ_COMMON
83cd4fe2
VP
654/*
655 * In the semi idle case, use the nearest busy cpu for migrating timers
656 * from an idle cpu. This is good for power-savings.
657 *
658 * We don't do similar optimization for completely idle system, as
659 * selecting an idle cpu will add more delays to the timers than intended
660 * (as that cpu's timer base may not be uptodate wrt jiffies etc).
661 */
6201b4d6 662int get_nohz_timer_target(int pinned)
83cd4fe2
VP
663{
664 int cpu = smp_processor_id();
665 int i;
666 struct sched_domain *sd;
667
6201b4d6
VK
668 if (pinned || !get_sysctl_timer_migration() || !idle_cpu(cpu))
669 return cpu;
670
057f3fad 671 rcu_read_lock();
83cd4fe2 672 for_each_domain(cpu, sd) {
057f3fad
PZ
673 for_each_cpu(i, sched_domain_span(sd)) {
674 if (!idle_cpu(i)) {
675 cpu = i;
676 goto unlock;
677 }
678 }
83cd4fe2 679 }
057f3fad
PZ
680unlock:
681 rcu_read_unlock();
83cd4fe2
VP
682 return cpu;
683}
06d8308c
TG
684/*
685 * When add_timer_on() enqueues a timer into the timer wheel of an
686 * idle CPU then this timer might expire before the next timer event
687 * which is scheduled to wake up that CPU. In case of a completely
688 * idle system the next event might even be infinite time into the
689 * future. wake_up_idle_cpu() ensures that the CPU is woken up and
690 * leaves the inner idle loop so the newly added timer is taken into
691 * account when the CPU goes back to idle and evaluates the timer
692 * wheel for the next timer event.
693 */
1c20091e 694static void wake_up_idle_cpu(int cpu)
06d8308c
TG
695{
696 struct rq *rq = cpu_rq(cpu);
697
698 if (cpu == smp_processor_id())
699 return;
700
67b9ca70 701 if (set_nr_and_not_polling(rq->idle))
06d8308c 702 smp_send_reschedule(cpu);
dfc68f29
AL
703 else
704 trace_sched_wake_idle_without_ipi(cpu);
45bf76df
IM
705}
706
c5bfece2 707static bool wake_up_full_nohz_cpu(int cpu)
1c20091e 708{
53c5fa16
FW
709 /*
710 * We just need the target to call irq_exit() and re-evaluate
711 * the next tick. The nohz full kick at least implies that.
712 * If needed we can still optimize that later with an
713 * empty IRQ.
714 */
c5bfece2 715 if (tick_nohz_full_cpu(cpu)) {
1c20091e
FW
716 if (cpu != smp_processor_id() ||
717 tick_nohz_tick_stopped())
53c5fa16 718 tick_nohz_full_kick_cpu(cpu);
1c20091e
FW
719 return true;
720 }
721
722 return false;
723}
724
725void wake_up_nohz_cpu(int cpu)
726{
c5bfece2 727 if (!wake_up_full_nohz_cpu(cpu))
1c20091e
FW
728 wake_up_idle_cpu(cpu);
729}
730
ca38062e 731static inline bool got_nohz_idle_kick(void)
45bf76df 732{
1c792db7 733 int cpu = smp_processor_id();
873b4c65
VG
734
735 if (!test_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu)))
736 return false;
737
738 if (idle_cpu(cpu) && !need_resched())
739 return true;
740
741 /*
742 * We can't run Idle Load Balance on this CPU for this time so we
743 * cancel it and clear NOHZ_BALANCE_KICK
744 */
745 clear_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu));
746 return false;
45bf76df
IM
747}
748
3451d024 749#else /* CONFIG_NO_HZ_COMMON */
45bf76df 750
ca38062e 751static inline bool got_nohz_idle_kick(void)
2069dd75 752{
ca38062e 753 return false;
2069dd75
PZ
754}
755
3451d024 756#endif /* CONFIG_NO_HZ_COMMON */
d842de87 757
ce831b38
FW
758#ifdef CONFIG_NO_HZ_FULL
759bool sched_can_stop_tick(void)
760{
3882ec64
FW
761 /*
762 * More than one running task need preemption.
763 * nr_running update is assumed to be visible
764 * after IPI is sent from wakers.
765 */
541b8264
VK
766 if (this_rq()->nr_running > 1)
767 return false;
ce831b38 768
541b8264 769 return true;
ce831b38
FW
770}
771#endif /* CONFIG_NO_HZ_FULL */
d842de87 772
029632fb 773void sched_avg_update(struct rq *rq)
18d95a28 774{
e9e9250b
PZ
775 s64 period = sched_avg_period();
776
78becc27 777 while ((s64)(rq_clock(rq) - rq->age_stamp) > period) {
0d98bb26
WD
778 /*
779 * Inline assembly required to prevent the compiler
780 * optimising this loop into a divmod call.
781 * See __iter_div_u64_rem() for another example of this.
782 */
783 asm("" : "+rm" (rq->age_stamp));
e9e9250b
PZ
784 rq->age_stamp += period;
785 rq->rt_avg /= 2;
786 }
18d95a28
PZ
787}
788
6d6bc0ad 789#endif /* CONFIG_SMP */
18d95a28 790
a790de99
PT
791#if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
792 (defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH)))
c09595f6 793/*
8277434e
PT
794 * Iterate task_group tree rooted at *from, calling @down when first entering a
795 * node and @up when leaving it for the final time.
796 *
797 * Caller must hold rcu_lock or sufficient equivalent.
c09595f6 798 */
029632fb 799int walk_tg_tree_from(struct task_group *from,
8277434e 800 tg_visitor down, tg_visitor up, void *data)
c09595f6
PZ
801{
802 struct task_group *parent, *child;
eb755805 803 int ret;
c09595f6 804
8277434e
PT
805 parent = from;
806
c09595f6 807down:
eb755805
PZ
808 ret = (*down)(parent, data);
809 if (ret)
8277434e 810 goto out;
c09595f6
PZ
811 list_for_each_entry_rcu(child, &parent->children, siblings) {
812 parent = child;
813 goto down;
814
815up:
816 continue;
817 }
eb755805 818 ret = (*up)(parent, data);
8277434e
PT
819 if (ret || parent == from)
820 goto out;
c09595f6
PZ
821
822 child = parent;
823 parent = parent->parent;
824 if (parent)
825 goto up;
8277434e 826out:
eb755805 827 return ret;
c09595f6
PZ
828}
829
029632fb 830int tg_nop(struct task_group *tg, void *data)
eb755805 831{
e2b245f8 832 return 0;
eb755805 833}
18d95a28
PZ
834#endif
835
45bf76df
IM
836static void set_load_weight(struct task_struct *p)
837{
f05998d4
NR
838 int prio = p->static_prio - MAX_RT_PRIO;
839 struct load_weight *load = &p->se.load;
840
dd41f596
IM
841 /*
842 * SCHED_IDLE tasks get minimal weight:
843 */
844 if (p->policy == SCHED_IDLE) {
c8b28116 845 load->weight = scale_load(WEIGHT_IDLEPRIO);
f05998d4 846 load->inv_weight = WMULT_IDLEPRIO;
dd41f596
IM
847 return;
848 }
71f8bd46 849
c8b28116 850 load->weight = scale_load(prio_to_weight[prio]);
f05998d4 851 load->inv_weight = prio_to_wmult[prio];
71f8bd46
IM
852}
853
371fd7e7 854static void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
2087a1ad 855{
a64692a3 856 update_rq_clock(rq);
43148951 857 sched_info_queued(rq, p);
371fd7e7 858 p->sched_class->enqueue_task(rq, p, flags);
71f8bd46
IM
859}
860
371fd7e7 861static void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
71f8bd46 862{
a64692a3 863 update_rq_clock(rq);
43148951 864 sched_info_dequeued(rq, p);
371fd7e7 865 p->sched_class->dequeue_task(rq, p, flags);
71f8bd46
IM
866}
867
029632fb 868void activate_task(struct rq *rq, struct task_struct *p, int flags)
1e3c88bd
PZ
869{
870 if (task_contributes_to_load(p))
871 rq->nr_uninterruptible--;
872
371fd7e7 873 enqueue_task(rq, p, flags);
1e3c88bd
PZ
874}
875
029632fb 876void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
1e3c88bd
PZ
877{
878 if (task_contributes_to_load(p))
879 rq->nr_uninterruptible++;
880
371fd7e7 881 dequeue_task(rq, p, flags);
1e3c88bd
PZ
882}
883
fe44d621 884static void update_rq_clock_task(struct rq *rq, s64 delta)
aa483808 885{
095c0aa8
GC
886/*
887 * In theory, the compile should just see 0 here, and optimize out the call
888 * to sched_rt_avg_update. But I don't trust it...
889 */
890#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
891 s64 steal = 0, irq_delta = 0;
892#endif
893#ifdef CONFIG_IRQ_TIME_ACCOUNTING
8e92c201 894 irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time;
fe44d621
PZ
895
896 /*
897 * Since irq_time is only updated on {soft,}irq_exit, we might run into
898 * this case when a previous update_rq_clock() happened inside a
899 * {soft,}irq region.
900 *
901 * When this happens, we stop ->clock_task and only update the
902 * prev_irq_time stamp to account for the part that fit, so that a next
903 * update will consume the rest. This ensures ->clock_task is
904 * monotonic.
905 *
906 * It does however cause some slight miss-attribution of {soft,}irq
907 * time, a more accurate solution would be to update the irq_time using
908 * the current rq->clock timestamp, except that would require using
909 * atomic ops.
910 */
911 if (irq_delta > delta)
912 irq_delta = delta;
913
914 rq->prev_irq_time += irq_delta;
915 delta -= irq_delta;
095c0aa8
GC
916#endif
917#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
c5905afb 918 if (static_key_false((&paravirt_steal_rq_enabled))) {
095c0aa8
GC
919 steal = paravirt_steal_clock(cpu_of(rq));
920 steal -= rq->prev_steal_time_rq;
921
922 if (unlikely(steal > delta))
923 steal = delta;
924
095c0aa8 925 rq->prev_steal_time_rq += steal;
095c0aa8
GC
926 delta -= steal;
927 }
928#endif
929
fe44d621
PZ
930 rq->clock_task += delta;
931
095c0aa8 932#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
5d4dfddd 933 if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY))
095c0aa8
GC
934 sched_rt_avg_update(rq, irq_delta + steal);
935#endif
aa483808
VP
936}
937
34f971f6
PZ
938void sched_set_stop_task(int cpu, struct task_struct *stop)
939{
940 struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
941 struct task_struct *old_stop = cpu_rq(cpu)->stop;
942
943 if (stop) {
944 /*
945 * Make it appear like a SCHED_FIFO task, its something
946 * userspace knows about and won't get confused about.
947 *
948 * Also, it will make PI more or less work without too
949 * much confusion -- but then, stop work should not
950 * rely on PI working anyway.
951 */
952 sched_setscheduler_nocheck(stop, SCHED_FIFO, &param);
953
954 stop->sched_class = &stop_sched_class;
955 }
956
957 cpu_rq(cpu)->stop = stop;
958
959 if (old_stop) {
960 /*
961 * Reset it back to a normal scheduling class so that
962 * it can die in pieces.
963 */
964 old_stop->sched_class = &rt_sched_class;
965 }
966}
967
14531189 968/*
dd41f596 969 * __normal_prio - return the priority that is based on the static prio
14531189 970 */
14531189
IM
971static inline int __normal_prio(struct task_struct *p)
972{
dd41f596 973 return p->static_prio;
14531189
IM
974}
975
b29739f9
IM
976/*
977 * Calculate the expected normal priority: i.e. priority
978 * without taking RT-inheritance into account. Might be
979 * boosted by interactivity modifiers. Changes upon fork,
980 * setprio syscalls, and whenever the interactivity
981 * estimator recalculates.
982 */
36c8b586 983static inline int normal_prio(struct task_struct *p)
b29739f9
IM
984{
985 int prio;
986
aab03e05
DF
987 if (task_has_dl_policy(p))
988 prio = MAX_DL_PRIO-1;
989 else if (task_has_rt_policy(p))
b29739f9
IM
990 prio = MAX_RT_PRIO-1 - p->rt_priority;
991 else
992 prio = __normal_prio(p);
993 return prio;
994}
995
996/*
997 * Calculate the current priority, i.e. the priority
998 * taken into account by the scheduler. This value might
999 * be boosted by RT tasks, or might be boosted by
1000 * interactivity modifiers. Will be RT if the task got
1001 * RT-boosted. If not then it returns p->normal_prio.
1002 */
36c8b586 1003static int effective_prio(struct task_struct *p)
b29739f9
IM
1004{
1005 p->normal_prio = normal_prio(p);
1006 /*
1007 * If we are RT tasks or we were boosted to RT priority,
1008 * keep the priority unchanged. Otherwise, update priority
1009 * to the normal priority:
1010 */
1011 if (!rt_prio(p->prio))
1012 return p->normal_prio;
1013 return p->prio;
1014}
1015
1da177e4
LT
1016/**
1017 * task_curr - is this task currently executing on a CPU?
1018 * @p: the task in question.
e69f6186
YB
1019 *
1020 * Return: 1 if the task is currently executing. 0 otherwise.
1da177e4 1021 */
36c8b586 1022inline int task_curr(const struct task_struct *p)
1da177e4
LT
1023{
1024 return cpu_curr(task_cpu(p)) == p;
1025}
1026
cb469845
SR
1027static inline void check_class_changed(struct rq *rq, struct task_struct *p,
1028 const struct sched_class *prev_class,
da7a735e 1029 int oldprio)
cb469845
SR
1030{
1031 if (prev_class != p->sched_class) {
1032 if (prev_class->switched_from)
da7a735e
PZ
1033 prev_class->switched_from(rq, p);
1034 p->sched_class->switched_to(rq, p);
2d3d891d 1035 } else if (oldprio != p->prio || dl_task(p))
da7a735e 1036 p->sched_class->prio_changed(rq, p, oldprio);
cb469845
SR
1037}
1038
029632fb 1039void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
1e5a7405
PZ
1040{
1041 const struct sched_class *class;
1042
1043 if (p->sched_class == rq->curr->sched_class) {
1044 rq->curr->sched_class->check_preempt_curr(rq, p, flags);
1045 } else {
1046 for_each_class(class) {
1047 if (class == rq->curr->sched_class)
1048 break;
1049 if (class == p->sched_class) {
8875125e 1050 resched_curr(rq);
1e5a7405
PZ
1051 break;
1052 }
1053 }
1054 }
1055
1056 /*
1057 * A queue event has occurred, and we're going to schedule. In
1058 * this case, we can save a useless back to back clock update.
1059 */
da0c1e65 1060 if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr))
1e5a7405
PZ
1061 rq->skip_clock_update = 1;
1062}
1063
1da177e4 1064#ifdef CONFIG_SMP
dd41f596 1065void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
c65cc870 1066{
e2912009
PZ
1067#ifdef CONFIG_SCHED_DEBUG
1068 /*
1069 * We should never call set_task_cpu() on a blocked task,
1070 * ttwu() will sort out the placement.
1071 */
077614ee 1072 WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
01028747 1073 !(task_preempt_count(p) & PREEMPT_ACTIVE));
0122ec5b
PZ
1074
1075#ifdef CONFIG_LOCKDEP
6c6c54e1
PZ
1076 /*
1077 * The caller should hold either p->pi_lock or rq->lock, when changing
1078 * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks.
1079 *
1080 * sched_move_task() holds both and thus holding either pins the cgroup,
8323f26c 1081 * see task_group().
6c6c54e1
PZ
1082 *
1083 * Furthermore, all task_rq users should acquire both locks, see
1084 * task_rq_lock().
1085 */
0122ec5b
PZ
1086 WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) ||
1087 lockdep_is_held(&task_rq(p)->lock)));
1088#endif
e2912009
PZ
1089#endif
1090
de1d7286 1091 trace_sched_migrate_task(p, new_cpu);
cbc34ed1 1092
0c69774e 1093 if (task_cpu(p) != new_cpu) {
0a74bef8
PT
1094 if (p->sched_class->migrate_task_rq)
1095 p->sched_class->migrate_task_rq(p, new_cpu);
0c69774e 1096 p->se.nr_migrations++;
a8b0ca17 1097 perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, NULL, 0);
0c69774e 1098 }
dd41f596
IM
1099
1100 __set_task_cpu(p, new_cpu);
c65cc870
IM
1101}
1102
ac66f547
PZ
1103static void __migrate_swap_task(struct task_struct *p, int cpu)
1104{
da0c1e65 1105 if (task_on_rq_queued(p)) {
ac66f547
PZ
1106 struct rq *src_rq, *dst_rq;
1107
1108 src_rq = task_rq(p);
1109 dst_rq = cpu_rq(cpu);
1110
1111 deactivate_task(src_rq, p, 0);
1112 set_task_cpu(p, cpu);
1113 activate_task(dst_rq, p, 0);
1114 check_preempt_curr(dst_rq, p, 0);
1115 } else {
1116 /*
1117 * Task isn't running anymore; make it appear like we migrated
1118 * it before it went to sleep. This means on wakeup we make the
1119 * previous cpu our targer instead of where it really is.
1120 */
1121 p->wake_cpu = cpu;
1122 }
1123}
1124
1125struct migration_swap_arg {
1126 struct task_struct *src_task, *dst_task;
1127 int src_cpu, dst_cpu;
1128};
1129
1130static int migrate_swap_stop(void *data)
1131{
1132 struct migration_swap_arg *arg = data;
1133 struct rq *src_rq, *dst_rq;
1134 int ret = -EAGAIN;
1135
1136 src_rq = cpu_rq(arg->src_cpu);
1137 dst_rq = cpu_rq(arg->dst_cpu);
1138
74602315
PZ
1139 double_raw_lock(&arg->src_task->pi_lock,
1140 &arg->dst_task->pi_lock);
ac66f547
PZ
1141 double_rq_lock(src_rq, dst_rq);
1142 if (task_cpu(arg->dst_task) != arg->dst_cpu)
1143 goto unlock;
1144
1145 if (task_cpu(arg->src_task) != arg->src_cpu)
1146 goto unlock;
1147
1148 if (!cpumask_test_cpu(arg->dst_cpu, tsk_cpus_allowed(arg->src_task)))
1149 goto unlock;
1150
1151 if (!cpumask_test_cpu(arg->src_cpu, tsk_cpus_allowed(arg->dst_task)))
1152 goto unlock;
1153
1154 __migrate_swap_task(arg->src_task, arg->dst_cpu);
1155 __migrate_swap_task(arg->dst_task, arg->src_cpu);
1156
1157 ret = 0;
1158
1159unlock:
1160 double_rq_unlock(src_rq, dst_rq);
74602315
PZ
1161 raw_spin_unlock(&arg->dst_task->pi_lock);
1162 raw_spin_unlock(&arg->src_task->pi_lock);
ac66f547
PZ
1163
1164 return ret;
1165}
1166
1167/*
1168 * Cross migrate two tasks
1169 */
1170int migrate_swap(struct task_struct *cur, struct task_struct *p)
1171{
1172 struct migration_swap_arg arg;
1173 int ret = -EINVAL;
1174
ac66f547
PZ
1175 arg = (struct migration_swap_arg){
1176 .src_task = cur,
1177 .src_cpu = task_cpu(cur),
1178 .dst_task = p,
1179 .dst_cpu = task_cpu(p),
1180 };
1181
1182 if (arg.src_cpu == arg.dst_cpu)
1183 goto out;
1184
6acce3ef
PZ
1185 /*
1186 * These three tests are all lockless; this is OK since all of them
1187 * will be re-checked with proper locks held further down the line.
1188 */
ac66f547
PZ
1189 if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu))
1190 goto out;
1191
1192 if (!cpumask_test_cpu(arg.dst_cpu, tsk_cpus_allowed(arg.src_task)))
1193 goto out;
1194
1195 if (!cpumask_test_cpu(arg.src_cpu, tsk_cpus_allowed(arg.dst_task)))
1196 goto out;
1197
286549dc 1198 trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu);
ac66f547
PZ
1199 ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg);
1200
1201out:
ac66f547
PZ
1202 return ret;
1203}
1204
969c7921 1205struct migration_arg {
36c8b586 1206 struct task_struct *task;
1da177e4 1207 int dest_cpu;
70b97a7f 1208};
1da177e4 1209
969c7921
TH
1210static int migration_cpu_stop(void *data);
1211
1da177e4
LT
1212/*
1213 * wait_task_inactive - wait for a thread to unschedule.
1214 *
85ba2d86
RM
1215 * If @match_state is nonzero, it's the @p->state value just checked and
1216 * not expected to change. If it changes, i.e. @p might have woken up,
1217 * then return zero. When we succeed in waiting for @p to be off its CPU,
1218 * we return a positive number (its total switch count). If a second call
1219 * a short while later returns the same number, the caller can be sure that
1220 * @p has remained unscheduled the whole time.
1221 *
1da177e4
LT
1222 * The caller must ensure that the task *will* unschedule sometime soon,
1223 * else this function might spin for a *long* time. This function can't
1224 * be called with interrupts off, or it may introduce deadlock with
1225 * smp_call_function() if an IPI is sent by the same process we are
1226 * waiting to become inactive.
1227 */
85ba2d86 1228unsigned long wait_task_inactive(struct task_struct *p, long match_state)
1da177e4
LT
1229{
1230 unsigned long flags;
da0c1e65 1231 int running, queued;
85ba2d86 1232 unsigned long ncsw;
70b97a7f 1233 struct rq *rq;
1da177e4 1234
3a5c359a
AK
1235 for (;;) {
1236 /*
1237 * We do the initial early heuristics without holding
1238 * any task-queue locks at all. We'll only try to get
1239 * the runqueue lock when things look like they will
1240 * work out!
1241 */
1242 rq = task_rq(p);
fa490cfd 1243
3a5c359a
AK
1244 /*
1245 * If the task is actively running on another CPU
1246 * still, just relax and busy-wait without holding
1247 * any locks.
1248 *
1249 * NOTE! Since we don't hold any locks, it's not
1250 * even sure that "rq" stays as the right runqueue!
1251 * But we don't care, since "task_running()" will
1252 * return false if the runqueue has changed and p
1253 * is actually now running somewhere else!
1254 */
85ba2d86
RM
1255 while (task_running(rq, p)) {
1256 if (match_state && unlikely(p->state != match_state))
1257 return 0;
3a5c359a 1258 cpu_relax();
85ba2d86 1259 }
fa490cfd 1260
3a5c359a
AK
1261 /*
1262 * Ok, time to look more closely! We need the rq
1263 * lock now, to be *sure*. If we're wrong, we'll
1264 * just go back and repeat.
1265 */
1266 rq = task_rq_lock(p, &flags);
27a9da65 1267 trace_sched_wait_task(p);
3a5c359a 1268 running = task_running(rq, p);
da0c1e65 1269 queued = task_on_rq_queued(p);
85ba2d86 1270 ncsw = 0;
f31e11d8 1271 if (!match_state || p->state == match_state)
93dcf55f 1272 ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
0122ec5b 1273 task_rq_unlock(rq, p, &flags);
fa490cfd 1274
85ba2d86
RM
1275 /*
1276 * If it changed from the expected state, bail out now.
1277 */
1278 if (unlikely(!ncsw))
1279 break;
1280
3a5c359a
AK
1281 /*
1282 * Was it really running after all now that we
1283 * checked with the proper locks actually held?
1284 *
1285 * Oops. Go back and try again..
1286 */
1287 if (unlikely(running)) {
1288 cpu_relax();
1289 continue;
1290 }
fa490cfd 1291
3a5c359a
AK
1292 /*
1293 * It's not enough that it's not actively running,
1294 * it must be off the runqueue _entirely_, and not
1295 * preempted!
1296 *
80dd99b3 1297 * So if it was still runnable (but just not actively
3a5c359a
AK
1298 * running right now), it's preempted, and we should
1299 * yield - it could be a while.
1300 */
da0c1e65 1301 if (unlikely(queued)) {
8eb90c30
TG
1302 ktime_t to = ktime_set(0, NSEC_PER_SEC/HZ);
1303
1304 set_current_state(TASK_UNINTERRUPTIBLE);
1305 schedule_hrtimeout(&to, HRTIMER_MODE_REL);
3a5c359a
AK
1306 continue;
1307 }
fa490cfd 1308
3a5c359a
AK
1309 /*
1310 * Ahh, all good. It wasn't running, and it wasn't
1311 * runnable, which means that it will never become
1312 * running in the future either. We're all done!
1313 */
1314 break;
1315 }
85ba2d86
RM
1316
1317 return ncsw;
1da177e4
LT
1318}
1319
1320/***
1321 * kick_process - kick a running thread to enter/exit the kernel
1322 * @p: the to-be-kicked thread
1323 *
1324 * Cause a process which is running on another CPU to enter
1325 * kernel-mode, without any delay. (to get signals handled.)
1326 *
25985edc 1327 * NOTE: this function doesn't have to take the runqueue lock,
1da177e4
LT
1328 * because all it wants to ensure is that the remote task enters
1329 * the kernel. If the IPI races and the task has been migrated
1330 * to another CPU then no harm is done and the purpose has been
1331 * achieved as well.
1332 */
36c8b586 1333void kick_process(struct task_struct *p)
1da177e4
LT
1334{
1335 int cpu;
1336
1337 preempt_disable();
1338 cpu = task_cpu(p);
1339 if ((cpu != smp_processor_id()) && task_curr(p))
1340 smp_send_reschedule(cpu);
1341 preempt_enable();
1342}
b43e3521 1343EXPORT_SYMBOL_GPL(kick_process);
476d139c 1344#endif /* CONFIG_SMP */
1da177e4 1345
970b13ba 1346#ifdef CONFIG_SMP
30da688e 1347/*
013fdb80 1348 * ->cpus_allowed is protected by both rq->lock and p->pi_lock
30da688e 1349 */
5da9a0fb
PZ
1350static int select_fallback_rq(int cpu, struct task_struct *p)
1351{
aa00d89c
TC
1352 int nid = cpu_to_node(cpu);
1353 const struct cpumask *nodemask = NULL;
2baab4e9
PZ
1354 enum { cpuset, possible, fail } state = cpuset;
1355 int dest_cpu;
5da9a0fb 1356
aa00d89c
TC
1357 /*
1358 * If the node that the cpu is on has been offlined, cpu_to_node()
1359 * will return -1. There is no cpu on the node, and we should
1360 * select the cpu on the other node.
1361 */
1362 if (nid != -1) {
1363 nodemask = cpumask_of_node(nid);
1364
1365 /* Look for allowed, online CPU in same node. */
1366 for_each_cpu(dest_cpu, nodemask) {
1367 if (!cpu_online(dest_cpu))
1368 continue;
1369 if (!cpu_active(dest_cpu))
1370 continue;
1371 if (cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p)))
1372 return dest_cpu;
1373 }
2baab4e9 1374 }
5da9a0fb 1375
2baab4e9
PZ
1376 for (;;) {
1377 /* Any allowed, online CPU? */
e3831edd 1378 for_each_cpu(dest_cpu, tsk_cpus_allowed(p)) {
2baab4e9
PZ
1379 if (!cpu_online(dest_cpu))
1380 continue;
1381 if (!cpu_active(dest_cpu))
1382 continue;
1383 goto out;
1384 }
5da9a0fb 1385
2baab4e9
PZ
1386 switch (state) {
1387 case cpuset:
1388 /* No more Mr. Nice Guy. */
1389 cpuset_cpus_allowed_fallback(p);
1390 state = possible;
1391 break;
1392
1393 case possible:
1394 do_set_cpus_allowed(p, cpu_possible_mask);
1395 state = fail;
1396 break;
1397
1398 case fail:
1399 BUG();
1400 break;
1401 }
1402 }
1403
1404out:
1405 if (state != cpuset) {
1406 /*
1407 * Don't tell them about moving exiting tasks or
1408 * kernel threads (both mm NULL), since they never
1409 * leave kernel.
1410 */
1411 if (p->mm && printk_ratelimit()) {
aac74dc4 1412 printk_deferred("process %d (%s) no longer affine to cpu%d\n",
2baab4e9
PZ
1413 task_pid_nr(p), p->comm, cpu);
1414 }
5da9a0fb
PZ
1415 }
1416
1417 return dest_cpu;
1418}
1419
e2912009 1420/*
013fdb80 1421 * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable.
e2912009 1422 */
970b13ba 1423static inline
ac66f547 1424int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)
970b13ba 1425{
ac66f547 1426 cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags);
e2912009
PZ
1427
1428 /*
1429 * In order not to call set_task_cpu() on a blocking task we need
1430 * to rely on ttwu() to place the task on a valid ->cpus_allowed
1431 * cpu.
1432 *
1433 * Since this is common to all placement strategies, this lives here.
1434 *
1435 * [ this allows ->select_task() to simply return task_cpu(p) and
1436 * not worry about this generic constraint ]
1437 */
fa17b507 1438 if (unlikely(!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)) ||
70f11205 1439 !cpu_online(cpu)))
5da9a0fb 1440 cpu = select_fallback_rq(task_cpu(p), p);
e2912009
PZ
1441
1442 return cpu;
970b13ba 1443}
09a40af5
MG
1444
1445static void update_avg(u64 *avg, u64 sample)
1446{
1447 s64 diff = sample - *avg;
1448 *avg += diff >> 3;
1449}
970b13ba
PZ
1450#endif
1451
d7c01d27 1452static void
b84cb5df 1453ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
9ed3811a 1454{
d7c01d27 1455#ifdef CONFIG_SCHEDSTATS
b84cb5df
PZ
1456 struct rq *rq = this_rq();
1457
d7c01d27
PZ
1458#ifdef CONFIG_SMP
1459 int this_cpu = smp_processor_id();
1460
1461 if (cpu == this_cpu) {
1462 schedstat_inc(rq, ttwu_local);
1463 schedstat_inc(p, se.statistics.nr_wakeups_local);
1464 } else {
1465 struct sched_domain *sd;
1466
1467 schedstat_inc(p, se.statistics.nr_wakeups_remote);
057f3fad 1468 rcu_read_lock();
d7c01d27
PZ
1469 for_each_domain(this_cpu, sd) {
1470 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
1471 schedstat_inc(sd, ttwu_wake_remote);
1472 break;
1473 }
1474 }
057f3fad 1475 rcu_read_unlock();
d7c01d27 1476 }
f339b9dc
PZ
1477
1478 if (wake_flags & WF_MIGRATED)
1479 schedstat_inc(p, se.statistics.nr_wakeups_migrate);
1480
d7c01d27
PZ
1481#endif /* CONFIG_SMP */
1482
1483 schedstat_inc(rq, ttwu_count);
9ed3811a 1484 schedstat_inc(p, se.statistics.nr_wakeups);
d7c01d27
PZ
1485
1486 if (wake_flags & WF_SYNC)
9ed3811a 1487 schedstat_inc(p, se.statistics.nr_wakeups_sync);
d7c01d27 1488
d7c01d27
PZ
1489#endif /* CONFIG_SCHEDSTATS */
1490}
1491
1492static void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags)
1493{
9ed3811a 1494 activate_task(rq, p, en_flags);
da0c1e65 1495 p->on_rq = TASK_ON_RQ_QUEUED;
c2f7115e
PZ
1496
1497 /* if a worker is waking up, notify workqueue */
1498 if (p->flags & PF_WQ_WORKER)
1499 wq_worker_waking_up(p, cpu_of(rq));
9ed3811a
TH
1500}
1501
23f41eeb
PZ
1502/*
1503 * Mark the task runnable and perform wakeup-preemption.
1504 */
89363381 1505static void
23f41eeb 1506ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags)
9ed3811a 1507{
9ed3811a 1508 check_preempt_curr(rq, p, wake_flags);
a8d7ad52 1509 trace_sched_wakeup(p, true);
9ed3811a
TH
1510
1511 p->state = TASK_RUNNING;
1512#ifdef CONFIG_SMP
1513 if (p->sched_class->task_woken)
1514 p->sched_class->task_woken(rq, p);
1515
e69c6341 1516 if (rq->idle_stamp) {
78becc27 1517 u64 delta = rq_clock(rq) - rq->idle_stamp;
9bd721c5 1518 u64 max = 2*rq->max_idle_balance_cost;
9ed3811a 1519
abfafa54
JL
1520 update_avg(&rq->avg_idle, delta);
1521
1522 if (rq->avg_idle > max)
9ed3811a 1523 rq->avg_idle = max;
abfafa54 1524
9ed3811a
TH
1525 rq->idle_stamp = 0;
1526 }
1527#endif
1528}
1529
c05fbafb
PZ
1530static void
1531ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags)
1532{
1533#ifdef CONFIG_SMP
1534 if (p->sched_contributes_to_load)
1535 rq->nr_uninterruptible--;
1536#endif
1537
1538 ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING);
1539 ttwu_do_wakeup(rq, p, wake_flags);
1540}
1541
1542/*
1543 * Called in case the task @p isn't fully descheduled from its runqueue,
1544 * in this case we must do a remote wakeup. Its a 'light' wakeup though,
1545 * since all we need to do is flip p->state to TASK_RUNNING, since
1546 * the task is still ->on_rq.
1547 */
1548static int ttwu_remote(struct task_struct *p, int wake_flags)
1549{
1550 struct rq *rq;
1551 int ret = 0;
1552
1553 rq = __task_rq_lock(p);
da0c1e65 1554 if (task_on_rq_queued(p)) {
1ad4ec0d
FW
1555 /* check_preempt_curr() may use rq clock */
1556 update_rq_clock(rq);
c05fbafb
PZ
1557 ttwu_do_wakeup(rq, p, wake_flags);
1558 ret = 1;
1559 }
1560 __task_rq_unlock(rq);
1561
1562 return ret;
1563}
1564
317f3941 1565#ifdef CONFIG_SMP
e3baac47 1566void sched_ttwu_pending(void)
317f3941
PZ
1567{
1568 struct rq *rq = this_rq();
fa14ff4a
PZ
1569 struct llist_node *llist = llist_del_all(&rq->wake_list);
1570 struct task_struct *p;
e3baac47 1571 unsigned long flags;
317f3941 1572
e3baac47
PZ
1573 if (!llist)
1574 return;
1575
1576 raw_spin_lock_irqsave(&rq->lock, flags);
317f3941 1577
fa14ff4a
PZ
1578 while (llist) {
1579 p = llist_entry(llist, struct task_struct, wake_entry);
1580 llist = llist_next(llist);
317f3941
PZ
1581 ttwu_do_activate(rq, p, 0);
1582 }
1583
e3baac47 1584 raw_spin_unlock_irqrestore(&rq->lock, flags);
317f3941
PZ
1585}
1586
1587void scheduler_ipi(void)
1588{
f27dde8d
PZ
1589 /*
1590 * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
1591 * TIF_NEED_RESCHED remotely (for the first time) will also send
1592 * this IPI.
1593 */
8cb75e0c 1594 preempt_fold_need_resched();
f27dde8d 1595
fd2ac4f4 1596 if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick())
c5d753a5
PZ
1597 return;
1598
1599 /*
1600 * Not all reschedule IPI handlers call irq_enter/irq_exit, since
1601 * traditionally all their work was done from the interrupt return
1602 * path. Now that we actually do some work, we need to make sure
1603 * we do call them.
1604 *
1605 * Some archs already do call them, luckily irq_enter/exit nest
1606 * properly.
1607 *
1608 * Arguably we should visit all archs and update all handlers,
1609 * however a fair share of IPIs are still resched only so this would
1610 * somewhat pessimize the simple resched case.
1611 */
1612 irq_enter();
fa14ff4a 1613 sched_ttwu_pending();
ca38062e
SS
1614
1615 /*
1616 * Check if someone kicked us for doing the nohz idle load balance.
1617 */
873b4c65 1618 if (unlikely(got_nohz_idle_kick())) {
6eb57e0d 1619 this_rq()->idle_balance = 1;
ca38062e 1620 raise_softirq_irqoff(SCHED_SOFTIRQ);
6eb57e0d 1621 }
c5d753a5 1622 irq_exit();
317f3941
PZ
1623}
1624
1625static void ttwu_queue_remote(struct task_struct *p, int cpu)
1626{
e3baac47
PZ
1627 struct rq *rq = cpu_rq(cpu);
1628
1629 if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) {
1630 if (!set_nr_if_polling(rq->idle))
1631 smp_send_reschedule(cpu);
1632 else
1633 trace_sched_wake_idle_without_ipi(cpu);
1634 }
317f3941 1635}
d6aa8f85 1636
f6be8af1
CL
1637void wake_up_if_idle(int cpu)
1638{
1639 struct rq *rq = cpu_rq(cpu);
1640 unsigned long flags;
1641
1642 if (!is_idle_task(rq->curr))
1643 return;
1644
1645 if (set_nr_if_polling(rq->idle)) {
1646 trace_sched_wake_idle_without_ipi(cpu);
1647 } else {
1648 raw_spin_lock_irqsave(&rq->lock, flags);
1649 if (is_idle_task(rq->curr))
1650 smp_send_reschedule(cpu);
1651 /* Else cpu is not in idle, do nothing here */
1652 raw_spin_unlock_irqrestore(&rq->lock, flags);
1653 }
1654}
1655
39be3501 1656bool cpus_share_cache(int this_cpu, int that_cpu)
518cd623
PZ
1657{
1658 return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
1659}
d6aa8f85 1660#endif /* CONFIG_SMP */
317f3941 1661
c05fbafb
PZ
1662static void ttwu_queue(struct task_struct *p, int cpu)
1663{
1664 struct rq *rq = cpu_rq(cpu);
1665
17d9f311 1666#if defined(CONFIG_SMP)
39be3501 1667 if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) {
f01114cb 1668 sched_clock_cpu(cpu); /* sync clocks x-cpu */
317f3941
PZ
1669 ttwu_queue_remote(p, cpu);
1670 return;
1671 }
1672#endif
1673
c05fbafb
PZ
1674 raw_spin_lock(&rq->lock);
1675 ttwu_do_activate(rq, p, 0);
1676 raw_spin_unlock(&rq->lock);
9ed3811a
TH
1677}
1678
1679/**
1da177e4 1680 * try_to_wake_up - wake up a thread
9ed3811a 1681 * @p: the thread to be awakened
1da177e4 1682 * @state: the mask of task states that can be woken
9ed3811a 1683 * @wake_flags: wake modifier flags (WF_*)
1da177e4
LT
1684 *
1685 * Put it on the run-queue if it's not already there. The "current"
1686 * thread is always on the run-queue (except when the actual
1687 * re-schedule is in progress), and as such you're allowed to do
1688 * the simpler "current->state = TASK_RUNNING" to mark yourself
1689 * runnable without the overhead of this.
1690 *
e69f6186 1691 * Return: %true if @p was woken up, %false if it was already running.
9ed3811a 1692 * or @state didn't match @p's state.
1da177e4 1693 */
e4a52bcb
PZ
1694static int
1695try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
1da177e4 1696{
1da177e4 1697 unsigned long flags;
c05fbafb 1698 int cpu, success = 0;
2398f2c6 1699
e0acd0a6
ON
1700 /*
1701 * If we are going to wake up a thread waiting for CONDITION we
1702 * need to ensure that CONDITION=1 done by the caller can not be
1703 * reordered with p->state check below. This pairs with mb() in
1704 * set_current_state() the waiting thread does.
1705 */
1706 smp_mb__before_spinlock();
013fdb80 1707 raw_spin_lock_irqsave(&p->pi_lock, flags);
e9c84311 1708 if (!(p->state & state))
1da177e4
LT
1709 goto out;
1710
c05fbafb 1711 success = 1; /* we're going to change ->state */
1da177e4 1712 cpu = task_cpu(p);
1da177e4 1713
cca26e80 1714 if (p->on_rq && ttwu_remote(p, wake_flags))
c05fbafb 1715 goto stat;
1da177e4 1716
1da177e4 1717#ifdef CONFIG_SMP
e9c84311 1718 /*
c05fbafb
PZ
1719 * If the owning (remote) cpu is still in the middle of schedule() with
1720 * this task as prev, wait until its done referencing the task.
e9c84311 1721 */
f3e94786 1722 while (p->on_cpu)
e4a52bcb 1723 cpu_relax();
0970d299 1724 /*
e4a52bcb 1725 * Pairs with the smp_wmb() in finish_lock_switch().
0970d299 1726 */
e4a52bcb 1727 smp_rmb();
1da177e4 1728
a8e4f2ea 1729 p->sched_contributes_to_load = !!task_contributes_to_load(p);
e9c84311 1730 p->state = TASK_WAKING;
e7693a36 1731
e4a52bcb 1732 if (p->sched_class->task_waking)
74f8e4b2 1733 p->sched_class->task_waking(p);
efbbd05a 1734
ac66f547 1735 cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags);
f339b9dc
PZ
1736 if (task_cpu(p) != cpu) {
1737 wake_flags |= WF_MIGRATED;
e4a52bcb 1738 set_task_cpu(p, cpu);
f339b9dc 1739 }
1da177e4 1740#endif /* CONFIG_SMP */
1da177e4 1741
c05fbafb
PZ
1742 ttwu_queue(p, cpu);
1743stat:
b84cb5df 1744 ttwu_stat(p, cpu, wake_flags);
1da177e4 1745out:
013fdb80 1746 raw_spin_unlock_irqrestore(&p->pi_lock, flags);
1da177e4
LT
1747
1748 return success;
1749}
1750
21aa9af0
TH
1751/**
1752 * try_to_wake_up_local - try to wake up a local task with rq lock held
1753 * @p: the thread to be awakened
1754 *
2acca55e 1755 * Put @p on the run-queue if it's not already there. The caller must
21aa9af0 1756 * ensure that this_rq() is locked, @p is bound to this_rq() and not
2acca55e 1757 * the current task.
21aa9af0
TH
1758 */
1759static void try_to_wake_up_local(struct task_struct *p)
1760{
1761 struct rq *rq = task_rq(p);
21aa9af0 1762
383efcd0
TH
1763 if (WARN_ON_ONCE(rq != this_rq()) ||
1764 WARN_ON_ONCE(p == current))
1765 return;
1766
21aa9af0
TH
1767 lockdep_assert_held(&rq->lock);
1768
2acca55e
PZ
1769 if (!raw_spin_trylock(&p->pi_lock)) {
1770 raw_spin_unlock(&rq->lock);
1771 raw_spin_lock(&p->pi_lock);
1772 raw_spin_lock(&rq->lock);
1773 }
1774
21aa9af0 1775 if (!(p->state & TASK_NORMAL))
2acca55e 1776 goto out;
21aa9af0 1777
da0c1e65 1778 if (!task_on_rq_queued(p))
d7c01d27
PZ
1779 ttwu_activate(rq, p, ENQUEUE_WAKEUP);
1780
23f41eeb 1781 ttwu_do_wakeup(rq, p, 0);
b84cb5df 1782 ttwu_stat(p, smp_processor_id(), 0);
2acca55e
PZ
1783out:
1784 raw_spin_unlock(&p->pi_lock);
21aa9af0
TH
1785}
1786
50fa610a
DH
1787/**
1788 * wake_up_process - Wake up a specific process
1789 * @p: The process to be woken up.
1790 *
1791 * Attempt to wake up the nominated process and move it to the set of runnable
e69f6186
YB
1792 * processes.
1793 *
1794 * Return: 1 if the process was woken up, 0 if it was already running.
50fa610a
DH
1795 *
1796 * It may be assumed that this function implies a write memory barrier before
1797 * changing the task state if and only if any tasks are woken up.
1798 */
7ad5b3a5 1799int wake_up_process(struct task_struct *p)
1da177e4 1800{
9067ac85
ON
1801 WARN_ON(task_is_stopped_or_traced(p));
1802 return try_to_wake_up(p, TASK_NORMAL, 0);
1da177e4 1803}
1da177e4
LT
1804EXPORT_SYMBOL(wake_up_process);
1805
7ad5b3a5 1806int wake_up_state(struct task_struct *p, unsigned int state)
1da177e4
LT
1807{
1808 return try_to_wake_up(p, state, 0);
1809}
1810
1da177e4
LT
1811/*
1812 * Perform scheduler related setup for a newly forked process p.
1813 * p is forked by current.
dd41f596
IM
1814 *
1815 * __sched_fork() is basic setup used by init_idle() too:
1816 */
5e1576ed 1817static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
dd41f596 1818{
fd2f4419
PZ
1819 p->on_rq = 0;
1820
1821 p->se.on_rq = 0;
dd41f596
IM
1822 p->se.exec_start = 0;
1823 p->se.sum_exec_runtime = 0;
f6cf891c 1824 p->se.prev_sum_exec_runtime = 0;
6c594c21 1825 p->se.nr_migrations = 0;
da7a735e 1826 p->se.vruntime = 0;
fd2f4419 1827 INIT_LIST_HEAD(&p->se.group_node);
6cfb0d5d
IM
1828
1829#ifdef CONFIG_SCHEDSTATS
41acab88 1830 memset(&p->se.statistics, 0, sizeof(p->se.statistics));
6cfb0d5d 1831#endif
476d139c 1832
aab03e05
DF
1833 RB_CLEAR_NODE(&p->dl.rb_node);
1834 hrtimer_init(&p->dl.dl_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1835 p->dl.dl_runtime = p->dl.runtime = 0;
1836 p->dl.dl_deadline = p->dl.deadline = 0;
755378a4 1837 p->dl.dl_period = 0;
aab03e05
DF
1838 p->dl.flags = 0;
1839
fa717060 1840 INIT_LIST_HEAD(&p->rt.run_list);
476d139c 1841
e107be36
AK
1842#ifdef CONFIG_PREEMPT_NOTIFIERS
1843 INIT_HLIST_HEAD(&p->preempt_notifiers);
1844#endif
cbee9f88
PZ
1845
1846#ifdef CONFIG_NUMA_BALANCING
1847 if (p->mm && atomic_read(&p->mm->mm_users) == 1) {
7e8d16b6 1848 p->mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
cbee9f88
PZ
1849 p->mm->numa_scan_seq = 0;
1850 }
1851
5e1576ed
RR
1852 if (clone_flags & CLONE_VM)
1853 p->numa_preferred_nid = current->numa_preferred_nid;
1854 else
1855 p->numa_preferred_nid = -1;
1856
cbee9f88
PZ
1857 p->node_stamp = 0ULL;
1858 p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0;
4b96a29b 1859 p->numa_scan_period = sysctl_numa_balancing_scan_delay;
cbee9f88 1860 p->numa_work.next = &p->numa_work;
ff1df896
RR
1861 p->numa_faults_memory = NULL;
1862 p->numa_faults_buffer_memory = NULL;
7e2703e6
RR
1863 p->last_task_numa_placement = 0;
1864 p->last_sum_exec_runtime = 0;
8c8a743c
PZ
1865
1866 INIT_LIST_HEAD(&p->numa_entry);
1867 p->numa_group = NULL;
cbee9f88 1868#endif /* CONFIG_NUMA_BALANCING */
dd41f596
IM
1869}
1870
1a687c2e 1871#ifdef CONFIG_NUMA_BALANCING
3105b86a 1872#ifdef CONFIG_SCHED_DEBUG
1a687c2e
MG
1873void set_numabalancing_state(bool enabled)
1874{
1875 if (enabled)
1876 sched_feat_set("NUMA");
1877 else
1878 sched_feat_set("NO_NUMA");
1879}
3105b86a
MG
1880#else
1881__read_mostly bool numabalancing_enabled;
1882
1883void set_numabalancing_state(bool enabled)
1884{
1885 numabalancing_enabled = enabled;
dd41f596 1886}
3105b86a 1887#endif /* CONFIG_SCHED_DEBUG */
54a43d54
AK
1888
1889#ifdef CONFIG_PROC_SYSCTL
1890int sysctl_numa_balancing(struct ctl_table *table, int write,
1891 void __user *buffer, size_t *lenp, loff_t *ppos)
1892{
1893 struct ctl_table t;
1894 int err;
1895 int state = numabalancing_enabled;
1896
1897 if (write && !capable(CAP_SYS_ADMIN))
1898 return -EPERM;
1899
1900 t = *table;
1901 t.data = &state;
1902 err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
1903 if (err < 0)
1904 return err;
1905 if (write)
1906 set_numabalancing_state(state);
1907 return err;
1908}
1909#endif
1910#endif
dd41f596
IM
1911
1912/*
1913 * fork()/clone()-time setup:
1914 */
aab03e05 1915int sched_fork(unsigned long clone_flags, struct task_struct *p)
dd41f596 1916{
0122ec5b 1917 unsigned long flags;
dd41f596
IM
1918 int cpu = get_cpu();
1919
5e1576ed 1920 __sched_fork(clone_flags, p);
06b83b5f 1921 /*
0017d735 1922 * We mark the process as running here. This guarantees that
06b83b5f
PZ
1923 * nobody will actually run it, and a signal or other external
1924 * event cannot wake it up and insert it on the runqueue either.
1925 */
0017d735 1926 p->state = TASK_RUNNING;
dd41f596 1927
c350a04e
MG
1928 /*
1929 * Make sure we do not leak PI boosting priority to the child.
1930 */
1931 p->prio = current->normal_prio;
1932
b9dc29e7
MG
1933 /*
1934 * Revert to default priority/policy on fork if requested.
1935 */
1936 if (unlikely(p->sched_reset_on_fork)) {
aab03e05 1937 if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
b9dc29e7 1938 p->policy = SCHED_NORMAL;
6c697bdf 1939 p->static_prio = NICE_TO_PRIO(0);
c350a04e
MG
1940 p->rt_priority = 0;
1941 } else if (PRIO_TO_NICE(p->static_prio) < 0)
1942 p->static_prio = NICE_TO_PRIO(0);
1943
1944 p->prio = p->normal_prio = __normal_prio(p);
1945 set_load_weight(p);
6c697bdf 1946
b9dc29e7
MG
1947 /*
1948 * We don't need the reset flag anymore after the fork. It has
1949 * fulfilled its duty:
1950 */
1951 p->sched_reset_on_fork = 0;
1952 }
ca94c442 1953
aab03e05
DF
1954 if (dl_prio(p->prio)) {
1955 put_cpu();
1956 return -EAGAIN;
1957 } else if (rt_prio(p->prio)) {
1958 p->sched_class = &rt_sched_class;
1959 } else {
2ddbf952 1960 p->sched_class = &fair_sched_class;
aab03e05 1961 }
b29739f9 1962
cd29fe6f
PZ
1963 if (p->sched_class->task_fork)
1964 p->sched_class->task_fork(p);
1965
86951599
PZ
1966 /*
1967 * The child is not yet in the pid-hash so no cgroup attach races,
1968 * and the cgroup is pinned to this child due to cgroup_fork()
1969 * is ran before sched_fork().
1970 *
1971 * Silence PROVE_RCU.
1972 */
0122ec5b 1973 raw_spin_lock_irqsave(&p->pi_lock, flags);
5f3edc1b 1974 set_task_cpu(p, cpu);
0122ec5b 1975 raw_spin_unlock_irqrestore(&p->pi_lock, flags);
5f3edc1b 1976
52f17b6c 1977#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
dd41f596 1978 if (likely(sched_info_on()))
52f17b6c 1979 memset(&p->sched_info, 0, sizeof(p->sched_info));
1da177e4 1980#endif
3ca7a440
PZ
1981#if defined(CONFIG_SMP)
1982 p->on_cpu = 0;
4866cde0 1983#endif
01028747 1984 init_task_preempt_count(p);
806c09a7 1985#ifdef CONFIG_SMP
917b627d 1986 plist_node_init(&p->pushable_tasks, MAX_PRIO);
1baca4ce 1987 RB_CLEAR_NODE(&p->pushable_dl_tasks);
806c09a7 1988#endif
917b627d 1989
476d139c 1990 put_cpu();
aab03e05 1991 return 0;
1da177e4
LT
1992}
1993
332ac17e
DF
1994unsigned long to_ratio(u64 period, u64 runtime)
1995{
1996 if (runtime == RUNTIME_INF)
1997 return 1ULL << 20;
1998
1999 /*
2000 * Doing this here saves a lot of checks in all
2001 * the calling paths, and returning zero seems
2002 * safe for them anyway.
2003 */
2004 if (period == 0)
2005 return 0;
2006
2007 return div64_u64(runtime << 20, period);
2008}
2009
2010#ifdef CONFIG_SMP
2011inline struct dl_bw *dl_bw_of(int i)
2012{
2013 return &cpu_rq(i)->rd->dl_bw;
2014}
2015
de212f18 2016static inline int dl_bw_cpus(int i)
332ac17e 2017{
de212f18
PZ
2018 struct root_domain *rd = cpu_rq(i)->rd;
2019 int cpus = 0;
2020
2021 for_each_cpu_and(i, rd->span, cpu_active_mask)
2022 cpus++;
2023
2024 return cpus;
332ac17e
DF
2025}
2026#else
2027inline struct dl_bw *dl_bw_of(int i)
2028{
2029 return &cpu_rq(i)->dl.dl_bw;
2030}
2031
de212f18 2032static inline int dl_bw_cpus(int i)
332ac17e
DF
2033{
2034 return 1;
2035}
2036#endif
2037
2038static inline
2039void __dl_clear(struct dl_bw *dl_b, u64 tsk_bw)
2040{
2041 dl_b->total_bw -= tsk_bw;
2042}
2043
2044static inline
2045void __dl_add(struct dl_bw *dl_b, u64 tsk_bw)
2046{
2047 dl_b->total_bw += tsk_bw;
2048}
2049
2050static inline
2051bool __dl_overflow(struct dl_bw *dl_b, int cpus, u64 old_bw, u64 new_bw)
2052{
2053 return dl_b->bw != -1 &&
2054 dl_b->bw * cpus < dl_b->total_bw - old_bw + new_bw;
2055}
2056
2057/*
2058 * We must be sure that accepting a new task (or allowing changing the
2059 * parameters of an existing one) is consistent with the bandwidth
2060 * constraints. If yes, this function also accordingly updates the currently
2061 * allocated bandwidth to reflect the new situation.
2062 *
2063 * This function is called while holding p's rq->lock.
2064 */
2065static int dl_overflow(struct task_struct *p, int policy,
2066 const struct sched_attr *attr)
2067{
2068
2069 struct dl_bw *dl_b = dl_bw_of(task_cpu(p));
4df1638c 2070 u64 period = attr->sched_period ?: attr->sched_deadline;
332ac17e
DF
2071 u64 runtime = attr->sched_runtime;
2072 u64 new_bw = dl_policy(policy) ? to_ratio(period, runtime) : 0;
de212f18 2073 int cpus, err = -1;
332ac17e
DF
2074
2075 if (new_bw == p->dl.dl_bw)
2076 return 0;
2077
2078 /*
2079 * Either if a task, enters, leave, or stays -deadline but changes
2080 * its parameters, we may need to update accordingly the total
2081 * allocated bandwidth of the container.
2082 */
2083 raw_spin_lock(&dl_b->lock);
de212f18 2084 cpus = dl_bw_cpus(task_cpu(p));
332ac17e
DF
2085 if (dl_policy(policy) && !task_has_dl_policy(p) &&
2086 !__dl_overflow(dl_b, cpus, 0, new_bw)) {
2087 __dl_add(dl_b, new_bw);
2088 err = 0;
2089 } else if (dl_policy(policy) && task_has_dl_policy(p) &&
2090 !__dl_overflow(dl_b, cpus, p->dl.dl_bw, new_bw)) {
2091 __dl_clear(dl_b, p->dl.dl_bw);
2092 __dl_add(dl_b, new_bw);
2093 err = 0;
2094 } else if (!dl_policy(policy) && task_has_dl_policy(p)) {
2095 __dl_clear(dl_b, p->dl.dl_bw);
2096 err = 0;
2097 }
2098 raw_spin_unlock(&dl_b->lock);
2099
2100 return err;
2101}
2102
2103extern void init_dl_bw(struct dl_bw *dl_b);
2104
1da177e4
LT
2105/*
2106 * wake_up_new_task - wake up a newly created task for the first time.
2107 *
2108 * This function will do some initial scheduler statistics housekeeping
2109 * that must be done for every newly created context, then puts the task
2110 * on the runqueue and wakes it.
2111 */
3e51e3ed 2112void wake_up_new_task(struct task_struct *p)
1da177e4
LT
2113{
2114 unsigned long flags;
dd41f596 2115 struct rq *rq;
fabf318e 2116
ab2515c4 2117 raw_spin_lock_irqsave(&p->pi_lock, flags);
fabf318e
PZ
2118#ifdef CONFIG_SMP
2119 /*
2120 * Fork balancing, do it here and not earlier because:
2121 * - cpus_allowed can change in the fork path
2122 * - any previously selected cpu might disappear through hotplug
fabf318e 2123 */
ac66f547 2124 set_task_cpu(p, select_task_rq(p, task_cpu(p), SD_BALANCE_FORK, 0));
0017d735
PZ
2125#endif
2126
a75cdaa9
AS
2127 /* Initialize new task's runnable average */
2128 init_task_runnable_average(p);
ab2515c4 2129 rq = __task_rq_lock(p);
cd29fe6f 2130 activate_task(rq, p, 0);
da0c1e65 2131 p->on_rq = TASK_ON_RQ_QUEUED;
89363381 2132 trace_sched_wakeup_new(p, true);
a7558e01 2133 check_preempt_curr(rq, p, WF_FORK);
9a897c5a 2134#ifdef CONFIG_SMP
efbbd05a
PZ
2135 if (p->sched_class->task_woken)
2136 p->sched_class->task_woken(rq, p);
9a897c5a 2137#endif
0122ec5b 2138 task_rq_unlock(rq, p, &flags);
1da177e4
LT
2139}
2140
e107be36
AK
2141#ifdef CONFIG_PREEMPT_NOTIFIERS
2142
2143/**
80dd99b3 2144 * preempt_notifier_register - tell me when current is being preempted & rescheduled
421cee29 2145 * @notifier: notifier struct to register
e107be36
AK
2146 */
2147void preempt_notifier_register(struct preempt_notifier *notifier)
2148{
2149 hlist_add_head(&notifier->link, &current->preempt_notifiers);
2150}
2151EXPORT_SYMBOL_GPL(preempt_notifier_register);
2152
2153/**
2154 * preempt_notifier_unregister - no longer interested in preemption notifications
421cee29 2155 * @notifier: notifier struct to unregister
e107be36
AK
2156 *
2157 * This is safe to call from within a preemption notifier.
2158 */
2159void preempt_notifier_unregister(struct preempt_notifier *notifier)
2160{
2161 hlist_del(&notifier->link);
2162}
2163EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
2164
2165static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
2166{
2167 struct preempt_notifier *notifier;
e107be36 2168
b67bfe0d 2169 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
e107be36
AK
2170 notifier->ops->sched_in(notifier, raw_smp_processor_id());
2171}
2172
2173static void
2174fire_sched_out_preempt_notifiers(struct task_struct *curr,
2175 struct task_struct *next)
2176{
2177 struct preempt_notifier *notifier;
e107be36 2178
b67bfe0d 2179 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
e107be36
AK
2180 notifier->ops->sched_out(notifier, next);
2181}
2182
6d6bc0ad 2183#else /* !CONFIG_PREEMPT_NOTIFIERS */
e107be36
AK
2184
2185static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
2186{
2187}
2188
2189static void
2190fire_sched_out_preempt_notifiers(struct task_struct *curr,
2191 struct task_struct *next)
2192{
2193}
2194
6d6bc0ad 2195#endif /* CONFIG_PREEMPT_NOTIFIERS */
e107be36 2196
4866cde0
NP
2197/**
2198 * prepare_task_switch - prepare to switch tasks
2199 * @rq: the runqueue preparing to switch
421cee29 2200 * @prev: the current task that is being switched out
4866cde0
NP
2201 * @next: the task we are going to switch to.
2202 *
2203 * This is called with the rq lock held and interrupts off. It must
2204 * be paired with a subsequent finish_task_switch after the context
2205 * switch.
2206 *
2207 * prepare_task_switch sets up locking and calls architecture specific
2208 * hooks.
2209 */
e107be36
AK
2210static inline void
2211prepare_task_switch(struct rq *rq, struct task_struct *prev,
2212 struct task_struct *next)
4866cde0 2213{
895dd92c 2214 trace_sched_switch(prev, next);
43148951 2215 sched_info_switch(rq, prev, next);
fe4b04fa 2216 perf_event_task_sched_out(prev, next);
e107be36 2217 fire_sched_out_preempt_notifiers(prev, next);
4866cde0
NP
2218 prepare_lock_switch(rq, next);
2219 prepare_arch_switch(next);
2220}
2221
1da177e4
LT
2222/**
2223 * finish_task_switch - clean up after a task-switch
344babaa 2224 * @rq: runqueue associated with task-switch
1da177e4
LT
2225 * @prev: the thread we just switched away from.
2226 *
4866cde0
NP
2227 * finish_task_switch must be called after the context switch, paired
2228 * with a prepare_task_switch call before the context switch.
2229 * finish_task_switch will reconcile locking set up by prepare_task_switch,
2230 * and do any other architecture-specific cleanup actions.
1da177e4
LT
2231 *
2232 * Note that we may have delayed dropping an mm in context_switch(). If
41a2d6cf 2233 * so, we finish that here outside of the runqueue lock. (Doing it
1da177e4
LT
2234 * with the lock held can cause deadlocks; see schedule() for
2235 * details.)
2236 */
a9957449 2237static void finish_task_switch(struct rq *rq, struct task_struct *prev)
1da177e4
LT
2238 __releases(rq->lock)
2239{
1da177e4 2240 struct mm_struct *mm = rq->prev_mm;
55a101f8 2241 long prev_state;
1da177e4
LT
2242
2243 rq->prev_mm = NULL;
2244
2245 /*
2246 * A task struct has one reference for the use as "current".
c394cc9f 2247 * If a task dies, then it sets TASK_DEAD in tsk->state and calls
55a101f8
ON
2248 * schedule one last time. The schedule call will never return, and
2249 * the scheduled task must drop that reference.
c394cc9f 2250 * The test for TASK_DEAD must occur while the runqueue locks are
1da177e4
LT
2251 * still held, otherwise prev could be scheduled on another cpu, die
2252 * there before we look at prev->state, and then the reference would
2253 * be dropped twice.
2254 * Manfred Spraul <manfred@colorfullife.com>
2255 */
55a101f8 2256 prev_state = prev->state;
bf9fae9f 2257 vtime_task_switch(prev);
4866cde0 2258 finish_arch_switch(prev);
a8d757ef 2259 perf_event_task_sched_in(prev, current);
4866cde0 2260 finish_lock_switch(rq, prev);
01f23e16 2261 finish_arch_post_lock_switch();
e8fa1362 2262
e107be36 2263 fire_sched_in_preempt_notifiers(current);
1da177e4
LT
2264 if (mm)
2265 mmdrop(mm);
c394cc9f 2266 if (unlikely(prev_state == TASK_DEAD)) {
e6c390f2
DF
2267 if (prev->sched_class->task_dead)
2268 prev->sched_class->task_dead(prev);
2269
c6fd91f0 2270 /*
2271 * Remove function-return probe instances associated with this
2272 * task and put them back on the free list.
9761eea8 2273 */
c6fd91f0 2274 kprobe_flush_task(prev);
1da177e4 2275 put_task_struct(prev);
c6fd91f0 2276 }
99e5ada9
FW
2277
2278 tick_nohz_task_switch(current);
1da177e4
LT
2279}
2280
3f029d3c
GH
2281#ifdef CONFIG_SMP
2282
3f029d3c
GH
2283/* rq->lock is NOT held, but preemption is disabled */
2284static inline void post_schedule(struct rq *rq)
2285{
2286 if (rq->post_schedule) {
2287 unsigned long flags;
2288
05fa785c 2289 raw_spin_lock_irqsave(&rq->lock, flags);
3f029d3c
GH
2290 if (rq->curr->sched_class->post_schedule)
2291 rq->curr->sched_class->post_schedule(rq);
05fa785c 2292 raw_spin_unlock_irqrestore(&rq->lock, flags);
3f029d3c
GH
2293
2294 rq->post_schedule = 0;
2295 }
2296}
2297
2298#else
da19ab51 2299
3f029d3c
GH
2300static inline void post_schedule(struct rq *rq)
2301{
1da177e4
LT
2302}
2303
3f029d3c
GH
2304#endif
2305
1da177e4
LT
2306/**
2307 * schedule_tail - first thing a freshly forked thread must call.
2308 * @prev: the thread we just switched away from.
2309 */
722a9f92 2310asmlinkage __visible void schedule_tail(struct task_struct *prev)
1da177e4
LT
2311 __releases(rq->lock)
2312{
70b97a7f
IM
2313 struct rq *rq = this_rq();
2314
4866cde0 2315 finish_task_switch(rq, prev);
da19ab51 2316
3f029d3c
GH
2317 /*
2318 * FIXME: do we need to worry about rq being invalidated by the
2319 * task_switch?
2320 */
2321 post_schedule(rq);
70b97a7f 2322
4866cde0
NP
2323#ifdef __ARCH_WANT_UNLOCKED_CTXSW
2324 /* In this case, finish_task_switch does not reenable preemption */
2325 preempt_enable();
2326#endif
1da177e4 2327 if (current->set_child_tid)
b488893a 2328 put_user(task_pid_vnr(current), current->set_child_tid);
1da177e4
LT
2329}
2330
2331/*
2332 * context_switch - switch to the new MM and the new
2333 * thread's register state.
2334 */
dd41f596 2335static inline void
70b97a7f 2336context_switch(struct rq *rq, struct task_struct *prev,
36c8b586 2337 struct task_struct *next)
1da177e4 2338{
dd41f596 2339 struct mm_struct *mm, *oldmm;
1da177e4 2340
e107be36 2341 prepare_task_switch(rq, prev, next);
fe4b04fa 2342
dd41f596
IM
2343 mm = next->mm;
2344 oldmm = prev->active_mm;
9226d125
ZA
2345 /*
2346 * For paravirt, this is coupled with an exit in switch_to to
2347 * combine the page table reload and the switch backend into
2348 * one hypercall.
2349 */
224101ed 2350 arch_start_context_switch(prev);
9226d125 2351
31915ab4 2352 if (!mm) {
1da177e4
LT
2353 next->active_mm = oldmm;
2354 atomic_inc(&oldmm->mm_count);
2355 enter_lazy_tlb(oldmm, next);
2356 } else
2357 switch_mm(oldmm, mm, next);
2358
31915ab4 2359 if (!prev->mm) {
1da177e4 2360 prev->active_mm = NULL;
1da177e4
LT
2361 rq->prev_mm = oldmm;
2362 }
3a5f5e48
IM
2363 /*
2364 * Since the runqueue lock will be released by the next
2365 * task (which is an invalid locking op but in the case
2366 * of the scheduler it's an obvious special-case), so we
2367 * do an early lockdep release here:
2368 */
2369#ifndef __ARCH_WANT_UNLOCKED_CTXSW
8a25d5de 2370 spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
3a5f5e48 2371#endif
1da177e4 2372
91d1aa43 2373 context_tracking_task_switch(prev, next);
1da177e4
LT
2374 /* Here we just switch the register state and the stack. */
2375 switch_to(prev, next, prev);
2376
dd41f596
IM
2377 barrier();
2378 /*
2379 * this_rq must be evaluated again because prev may have moved
2380 * CPUs since it called schedule(), thus the 'rq' on its stack
2381 * frame will be invalid.
2382 */
2383 finish_task_switch(this_rq(), prev);
1da177e4
LT
2384}
2385
2386/*
1c3e8264 2387 * nr_running and nr_context_switches:
1da177e4
LT
2388 *
2389 * externally visible scheduler statistics: current number of runnable
1c3e8264 2390 * threads, total number of context switches performed since bootup.
1da177e4
LT
2391 */
2392unsigned long nr_running(void)
2393{
2394 unsigned long i, sum = 0;
2395
2396 for_each_online_cpu(i)
2397 sum += cpu_rq(i)->nr_running;
2398
2399 return sum;
f711f609 2400}
1da177e4 2401
1da177e4 2402unsigned long long nr_context_switches(void)
46cb4b7c 2403{
cc94abfc
SR
2404 int i;
2405 unsigned long long sum = 0;
46cb4b7c 2406
0a945022 2407 for_each_possible_cpu(i)
1da177e4 2408 sum += cpu_rq(i)->nr_switches;
46cb4b7c 2409
1da177e4
LT
2410 return sum;
2411}
483b4ee6 2412
1da177e4
LT
2413unsigned long nr_iowait(void)
2414{
2415 unsigned long i, sum = 0;
483b4ee6 2416
0a945022 2417 for_each_possible_cpu(i)
1da177e4 2418 sum += atomic_read(&cpu_rq(i)->nr_iowait);
46cb4b7c 2419
1da177e4
LT
2420 return sum;
2421}
483b4ee6 2422
8c215bd3 2423unsigned long nr_iowait_cpu(int cpu)
69d25870 2424{
8c215bd3 2425 struct rq *this = cpu_rq(cpu);
69d25870
AV
2426 return atomic_read(&this->nr_iowait);
2427}
46cb4b7c 2428
372ba8cb
MG
2429void get_iowait_load(unsigned long *nr_waiters, unsigned long *load)
2430{
2431 struct rq *this = this_rq();
2432 *nr_waiters = atomic_read(&this->nr_iowait);
2433 *load = this->cpu_load[0];
2434}
2435
dd41f596 2436#ifdef CONFIG_SMP
8a0be9ef 2437
46cb4b7c 2438/*
38022906
PZ
2439 * sched_exec - execve() is a valuable balancing opportunity, because at
2440 * this point the task has the smallest effective memory and cache footprint.
46cb4b7c 2441 */
38022906 2442void sched_exec(void)
46cb4b7c 2443{
38022906 2444 struct task_struct *p = current;
1da177e4 2445 unsigned long flags;
0017d735 2446 int dest_cpu;
46cb4b7c 2447
8f42ced9 2448 raw_spin_lock_irqsave(&p->pi_lock, flags);
ac66f547 2449 dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0);
0017d735
PZ
2450 if (dest_cpu == smp_processor_id())
2451 goto unlock;
38022906 2452
8f42ced9 2453 if (likely(cpu_active(dest_cpu))) {
969c7921 2454 struct migration_arg arg = { p, dest_cpu };
46cb4b7c 2455
8f42ced9
PZ
2456 raw_spin_unlock_irqrestore(&p->pi_lock, flags);
2457 stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg);
1da177e4
LT
2458 return;
2459 }
0017d735 2460unlock:
8f42ced9 2461 raw_spin_unlock_irqrestore(&p->pi_lock, flags);
1da177e4 2462}
dd41f596 2463
1da177e4
LT
2464#endif
2465
1da177e4 2466DEFINE_PER_CPU(struct kernel_stat, kstat);
3292beb3 2467DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat);
1da177e4
LT
2468
2469EXPORT_PER_CPU_SYMBOL(kstat);
3292beb3 2470EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
1da177e4
LT
2471
2472/*
c5f8d995 2473 * Return any ns on the sched_clock that have not yet been accounted in
f06febc9 2474 * @p in case that task is currently running.
c5f8d995
HS
2475 *
2476 * Called with task_rq_lock() held on @rq.
1da177e4 2477 */
c5f8d995
HS
2478static u64 do_task_delta_exec(struct task_struct *p, struct rq *rq)
2479{
2480 u64 ns = 0;
2481
4036ac15
MG
2482 /*
2483 * Must be ->curr _and_ ->on_rq. If dequeued, we would
2484 * project cycles that may never be accounted to this
2485 * thread, breaking clock_gettime().
2486 */
da0c1e65 2487 if (task_current(rq, p) && task_on_rq_queued(p)) {
c5f8d995 2488 update_rq_clock(rq);
78becc27 2489 ns = rq_clock_task(rq) - p->se.exec_start;
c5f8d995
HS
2490 if ((s64)ns < 0)
2491 ns = 0;
2492 }
2493
2494 return ns;
2495}
2496
bb34d92f 2497unsigned long long task_delta_exec(struct task_struct *p)
1da177e4 2498{
1da177e4 2499 unsigned long flags;
41b86e9c 2500 struct rq *rq;
bb34d92f 2501 u64 ns = 0;
48f24c4d 2502
41b86e9c 2503 rq = task_rq_lock(p, &flags);
c5f8d995 2504 ns = do_task_delta_exec(p, rq);
0122ec5b 2505 task_rq_unlock(rq, p, &flags);
1508487e 2506
c5f8d995
HS
2507 return ns;
2508}
f06febc9 2509
c5f8d995
HS
2510/*
2511 * Return accounted runtime for the task.
2512 * In case the task is currently running, return the runtime plus current's
2513 * pending runtime that have not been accounted yet.
2514 */
2515unsigned long long task_sched_runtime(struct task_struct *p)
2516{
2517 unsigned long flags;
2518 struct rq *rq;
2519 u64 ns = 0;
2520
911b2898
PZ
2521#if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
2522 /*
2523 * 64-bit doesn't need locks to atomically read a 64bit value.
2524 * So we have a optimization chance when the task's delta_exec is 0.
2525 * Reading ->on_cpu is racy, but this is ok.
2526 *
2527 * If we race with it leaving cpu, we'll take a lock. So we're correct.
2528 * If we race with it entering cpu, unaccounted time is 0. This is
2529 * indistinguishable from the read occurring a few cycles earlier.
4036ac15
MG
2530 * If we see ->on_cpu without ->on_rq, the task is leaving, and has
2531 * been accounted, so we're correct here as well.
911b2898 2532 */
da0c1e65 2533 if (!p->on_cpu || !task_on_rq_queued(p))
911b2898
PZ
2534 return p->se.sum_exec_runtime;
2535#endif
2536
c5f8d995
HS
2537 rq = task_rq_lock(p, &flags);
2538 ns = p->se.sum_exec_runtime + do_task_delta_exec(p, rq);
0122ec5b 2539 task_rq_unlock(rq, p, &flags);
c5f8d995
HS
2540
2541 return ns;
2542}
48f24c4d 2543
7835b98b
CL
2544/*
2545 * This function gets called by the timer code, with HZ frequency.
2546 * We call it with interrupts disabled.
7835b98b
CL
2547 */
2548void scheduler_tick(void)
2549{
7835b98b
CL
2550 int cpu = smp_processor_id();
2551 struct rq *rq = cpu_rq(cpu);
dd41f596 2552 struct task_struct *curr = rq->curr;
3e51f33f
PZ
2553
2554 sched_clock_tick();
dd41f596 2555
05fa785c 2556 raw_spin_lock(&rq->lock);
3e51f33f 2557 update_rq_clock(rq);
fa85ae24 2558 curr->sched_class->task_tick(rq, curr, 0);
83dfd523 2559 update_cpu_load_active(rq);
05fa785c 2560 raw_spin_unlock(&rq->lock);
7835b98b 2561
e9d2b064 2562 perf_event_task_tick();
e220d2dc 2563
e418e1c2 2564#ifdef CONFIG_SMP
6eb57e0d 2565 rq->idle_balance = idle_cpu(cpu);
7caff66f 2566 trigger_load_balance(rq);
e418e1c2 2567#endif
265f22a9 2568 rq_last_tick_reset(rq);
1da177e4
LT
2569}
2570
265f22a9
FW
2571#ifdef CONFIG_NO_HZ_FULL
2572/**
2573 * scheduler_tick_max_deferment
2574 *
2575 * Keep at least one tick per second when a single
2576 * active task is running because the scheduler doesn't
2577 * yet completely support full dynticks environment.
2578 *
2579 * This makes sure that uptime, CFS vruntime, load
2580 * balancing, etc... continue to move forward, even
2581 * with a very low granularity.
e69f6186
YB
2582 *
2583 * Return: Maximum deferment in nanoseconds.
265f22a9
FW
2584 */
2585u64 scheduler_tick_max_deferment(void)
2586{
2587 struct rq *rq = this_rq();
2588 unsigned long next, now = ACCESS_ONCE(jiffies);
2589
2590 next = rq->last_sched_tick + HZ;
2591
2592 if (time_before_eq(next, now))
2593 return 0;
2594
8fe8ff09 2595 return jiffies_to_nsecs(next - now);
1da177e4 2596}
265f22a9 2597#endif
1da177e4 2598
132380a0 2599notrace unsigned long get_parent_ip(unsigned long addr)
6cd8a4bb
SR
2600{
2601 if (in_lock_functions(addr)) {
2602 addr = CALLER_ADDR2;
2603 if (in_lock_functions(addr))
2604 addr = CALLER_ADDR3;
2605 }
2606 return addr;
2607}
1da177e4 2608
7e49fcce
SR
2609#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
2610 defined(CONFIG_PREEMPT_TRACER))
2611
edafe3a5 2612void preempt_count_add(int val)
1da177e4 2613{
6cd8a4bb 2614#ifdef CONFIG_DEBUG_PREEMPT
1da177e4
LT
2615 /*
2616 * Underflow?
2617 */
9a11b49a
IM
2618 if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
2619 return;
6cd8a4bb 2620#endif
bdb43806 2621 __preempt_count_add(val);
6cd8a4bb 2622#ifdef CONFIG_DEBUG_PREEMPT
1da177e4
LT
2623 /*
2624 * Spinlock count overflowing soon?
2625 */
33859f7f
MOS
2626 DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
2627 PREEMPT_MASK - 10);
6cd8a4bb 2628#endif
8f47b187
TG
2629 if (preempt_count() == val) {
2630 unsigned long ip = get_parent_ip(CALLER_ADDR1);
2631#ifdef CONFIG_DEBUG_PREEMPT
2632 current->preempt_disable_ip = ip;
2633#endif
2634 trace_preempt_off(CALLER_ADDR0, ip);
2635 }
1da177e4 2636}
bdb43806 2637EXPORT_SYMBOL(preempt_count_add);
edafe3a5 2638NOKPROBE_SYMBOL(preempt_count_add);
1da177e4 2639
edafe3a5 2640void preempt_count_sub(int val)
1da177e4 2641{
6cd8a4bb 2642#ifdef CONFIG_DEBUG_PREEMPT
1da177e4
LT
2643 /*
2644 * Underflow?
2645 */
01e3eb82 2646 if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
9a11b49a 2647 return;
1da177e4
LT
2648 /*
2649 * Is the spinlock portion underflowing?
2650 */
9a11b49a
IM
2651 if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
2652 !(preempt_count() & PREEMPT_MASK)))
2653 return;
6cd8a4bb 2654#endif
9a11b49a 2655
6cd8a4bb
SR
2656 if (preempt_count() == val)
2657 trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
bdb43806 2658 __preempt_count_sub(val);
1da177e4 2659}
bdb43806 2660EXPORT_SYMBOL(preempt_count_sub);
edafe3a5 2661NOKPROBE_SYMBOL(preempt_count_sub);
1da177e4
LT
2662
2663#endif
2664
2665/*
dd41f596 2666 * Print scheduling while atomic bug:
1da177e4 2667 */
dd41f596 2668static noinline void __schedule_bug(struct task_struct *prev)
1da177e4 2669{
664dfa65
DJ
2670 if (oops_in_progress)
2671 return;
2672
3df0fc5b
PZ
2673 printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
2674 prev->comm, prev->pid, preempt_count());
838225b4 2675
dd41f596 2676 debug_show_held_locks(prev);
e21f5b15 2677 print_modules();
dd41f596
IM
2678 if (irqs_disabled())
2679 print_irqtrace_events(prev);
8f47b187
TG
2680#ifdef CONFIG_DEBUG_PREEMPT
2681 if (in_atomic_preempt_off()) {
2682 pr_err("Preemption disabled at:");
2683 print_ip_sym(current->preempt_disable_ip);
2684 pr_cont("\n");
2685 }
2686#endif
6135fc1e 2687 dump_stack();
373d4d09 2688 add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
dd41f596 2689}
1da177e4 2690
dd41f596
IM
2691/*
2692 * Various schedule()-time debugging checks and statistics:
2693 */
2694static inline void schedule_debug(struct task_struct *prev)
2695{
0d9e2632
AT
2696#ifdef CONFIG_SCHED_STACK_END_CHECK
2697 BUG_ON(unlikely(task_stack_end_corrupted(prev)));
2698#endif
1da177e4 2699 /*
41a2d6cf 2700 * Test if we are atomic. Since do_exit() needs to call into
192301e7
ON
2701 * schedule() atomically, we ignore that path. Otherwise whine
2702 * if we are scheduling when we should not.
1da177e4 2703 */
192301e7 2704 if (unlikely(in_atomic_preempt_off() && prev->state != TASK_DEAD))
dd41f596 2705 __schedule_bug(prev);
b3fbab05 2706 rcu_sleep_check();
dd41f596 2707
1da177e4
LT
2708 profile_hit(SCHED_PROFILING, __builtin_return_address(0));
2709
2d72376b 2710 schedstat_inc(this_rq(), sched_count);
dd41f596
IM
2711}
2712
2713/*
2714 * Pick up the highest-prio task:
2715 */
2716static inline struct task_struct *
606dba2e 2717pick_next_task(struct rq *rq, struct task_struct *prev)
dd41f596 2718{
37e117c0 2719 const struct sched_class *class = &fair_sched_class;
dd41f596 2720 struct task_struct *p;
1da177e4
LT
2721
2722 /*
dd41f596
IM
2723 * Optimization: we know that if all tasks are in
2724 * the fair class we can call that function directly:
1da177e4 2725 */
37e117c0 2726 if (likely(prev->sched_class == class &&
38033c37 2727 rq->nr_running == rq->cfs.h_nr_running)) {
606dba2e 2728 p = fair_sched_class.pick_next_task(rq, prev);
6ccdc84b
PZ
2729 if (unlikely(p == RETRY_TASK))
2730 goto again;
2731
2732 /* assumes fair_sched_class->next == idle_sched_class */
2733 if (unlikely(!p))
2734 p = idle_sched_class.pick_next_task(rq, prev);
2735
2736 return p;
1da177e4
LT
2737 }
2738
37e117c0 2739again:
34f971f6 2740 for_each_class(class) {
606dba2e 2741 p = class->pick_next_task(rq, prev);
37e117c0
PZ
2742 if (p) {
2743 if (unlikely(p == RETRY_TASK))
2744 goto again;
dd41f596 2745 return p;
37e117c0 2746 }
dd41f596 2747 }
34f971f6
PZ
2748
2749 BUG(); /* the idle class will always have a runnable task */
dd41f596 2750}
1da177e4 2751
dd41f596 2752/*
c259e01a 2753 * __schedule() is the main scheduler function.
edde96ea
PE
2754 *
2755 * The main means of driving the scheduler and thus entering this function are:
2756 *
2757 * 1. Explicit blocking: mutex, semaphore, waitqueue, etc.
2758 *
2759 * 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return
2760 * paths. For example, see arch/x86/entry_64.S.
2761 *
2762 * To drive preemption between tasks, the scheduler sets the flag in timer
2763 * interrupt handler scheduler_tick().
2764 *
2765 * 3. Wakeups don't really cause entry into schedule(). They add a
2766 * task to the run-queue and that's it.
2767 *
2768 * Now, if the new task added to the run-queue preempts the current
2769 * task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets
2770 * called on the nearest possible occasion:
2771 *
2772 * - If the kernel is preemptible (CONFIG_PREEMPT=y):
2773 *
2774 * - in syscall or exception context, at the next outmost
2775 * preempt_enable(). (this might be as soon as the wake_up()'s
2776 * spin_unlock()!)
2777 *
2778 * - in IRQ context, return from interrupt-handler to
2779 * preemptible context
2780 *
2781 * - If the kernel is not preemptible (CONFIG_PREEMPT is not set)
2782 * then at the next:
2783 *
2784 * - cond_resched() call
2785 * - explicit schedule() call
2786 * - return from syscall or exception to user-space
2787 * - return from interrupt-handler to user-space
dd41f596 2788 */
c259e01a 2789static void __sched __schedule(void)
dd41f596
IM
2790{
2791 struct task_struct *prev, *next;
67ca7bde 2792 unsigned long *switch_count;
dd41f596 2793 struct rq *rq;
31656519 2794 int cpu;
dd41f596 2795
ff743345
PZ
2796need_resched:
2797 preempt_disable();
dd41f596
IM
2798 cpu = smp_processor_id();
2799 rq = cpu_rq(cpu);
25502a6c 2800 rcu_note_context_switch(cpu);
dd41f596 2801 prev = rq->curr;
dd41f596 2802
dd41f596 2803 schedule_debug(prev);
1da177e4 2804
31656519 2805 if (sched_feat(HRTICK))
f333fdc9 2806 hrtick_clear(rq);
8f4d37ec 2807
e0acd0a6
ON
2808 /*
2809 * Make sure that signal_pending_state()->signal_pending() below
2810 * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE)
2811 * done by the caller to avoid the race with signal_wake_up().
2812 */
2813 smp_mb__before_spinlock();
05fa785c 2814 raw_spin_lock_irq(&rq->lock);
1da177e4 2815
246d86b5 2816 switch_count = &prev->nivcsw;
1da177e4 2817 if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
21aa9af0 2818 if (unlikely(signal_pending_state(prev->state, prev))) {
1da177e4 2819 prev->state = TASK_RUNNING;
21aa9af0 2820 } else {
2acca55e
PZ
2821 deactivate_task(rq, prev, DEQUEUE_SLEEP);
2822 prev->on_rq = 0;
2823
21aa9af0 2824 /*
2acca55e
PZ
2825 * If a worker went to sleep, notify and ask workqueue
2826 * whether it wants to wake up a task to maintain
2827 * concurrency.
21aa9af0
TH
2828 */
2829 if (prev->flags & PF_WQ_WORKER) {
2830 struct task_struct *to_wakeup;
2831
2832 to_wakeup = wq_worker_sleeping(prev, cpu);
2833 if (to_wakeup)
2834 try_to_wake_up_local(to_wakeup);
2835 }
21aa9af0 2836 }
dd41f596 2837 switch_count = &prev->nvcsw;
1da177e4
LT
2838 }
2839
da0c1e65 2840 if (task_on_rq_queued(prev) || rq->skip_clock_update < 0)
606dba2e
PZ
2841 update_rq_clock(rq);
2842
2843 next = pick_next_task(rq, prev);
f26f9aff 2844 clear_tsk_need_resched(prev);
f27dde8d 2845 clear_preempt_need_resched();
f26f9aff 2846 rq->skip_clock_update = 0;
1da177e4 2847
1da177e4 2848 if (likely(prev != next)) {
1da177e4
LT
2849 rq->nr_switches++;
2850 rq->curr = next;
2851 ++*switch_count;
2852
dd41f596 2853 context_switch(rq, prev, next); /* unlocks the rq */
8f4d37ec 2854 /*
246d86b5
ON
2855 * The context switch have flipped the stack from under us
2856 * and restored the local variables which were saved when
2857 * this task called schedule() in the past. prev == current
2858 * is still correct, but it can be moved to another cpu/rq.
8f4d37ec
PZ
2859 */
2860 cpu = smp_processor_id();
2861 rq = cpu_rq(cpu);
1da177e4 2862 } else
05fa785c 2863 raw_spin_unlock_irq(&rq->lock);
1da177e4 2864
3f029d3c 2865 post_schedule(rq);
1da177e4 2866
ba74c144 2867 sched_preempt_enable_no_resched();
ff743345 2868 if (need_resched())
1da177e4
LT
2869 goto need_resched;
2870}
c259e01a 2871
9c40cef2
TG
2872static inline void sched_submit_work(struct task_struct *tsk)
2873{
3c7d5184 2874 if (!tsk->state || tsk_is_pi_blocked(tsk))
9c40cef2
TG
2875 return;
2876 /*
2877 * If we are going to sleep and we have plugged IO queued,
2878 * make sure to submit it to avoid deadlocks.
2879 */
2880 if (blk_needs_flush_plug(tsk))
2881 blk_schedule_flush_plug(tsk);
2882}
2883
722a9f92 2884asmlinkage __visible void __sched schedule(void)
c259e01a 2885{
9c40cef2
TG
2886 struct task_struct *tsk = current;
2887
2888 sched_submit_work(tsk);
c259e01a
TG
2889 __schedule();
2890}
1da177e4
LT
2891EXPORT_SYMBOL(schedule);
2892
91d1aa43 2893#ifdef CONFIG_CONTEXT_TRACKING
722a9f92 2894asmlinkage __visible void __sched schedule_user(void)
20ab65e3
FW
2895{
2896 /*
2897 * If we come here after a random call to set_need_resched(),
2898 * or we have been woken up remotely but the IPI has not yet arrived,
2899 * we haven't yet exited the RCU idle mode. Do it here manually until
2900 * we find a better solution.
2901 */
91d1aa43 2902 user_exit();
20ab65e3 2903 schedule();
91d1aa43 2904 user_enter();
20ab65e3
FW
2905}
2906#endif
2907
c5491ea7
TG
2908/**
2909 * schedule_preempt_disabled - called with preemption disabled
2910 *
2911 * Returns with preemption disabled. Note: preempt_count must be 1
2912 */
2913void __sched schedule_preempt_disabled(void)
2914{
ba74c144 2915 sched_preempt_enable_no_resched();
c5491ea7
TG
2916 schedule();
2917 preempt_disable();
2918}
2919
1da177e4
LT
2920#ifdef CONFIG_PREEMPT
2921/*
2ed6e34f 2922 * this is the entry point to schedule() from in-kernel preemption
41a2d6cf 2923 * off of preempt_enable. Kernel preemptions off return from interrupt
1da177e4
LT
2924 * occur there and call schedule directly.
2925 */
722a9f92 2926asmlinkage __visible void __sched notrace preempt_schedule(void)
1da177e4 2927{
1da177e4
LT
2928 /*
2929 * If there is a non-zero preempt_count or interrupts are disabled,
41a2d6cf 2930 * we do not want to preempt the current task. Just return..
1da177e4 2931 */
fbb00b56 2932 if (likely(!preemptible()))
1da177e4
LT
2933 return;
2934
3a5c359a 2935 do {
bdb43806 2936 __preempt_count_add(PREEMPT_ACTIVE);
c259e01a 2937 __schedule();
bdb43806 2938 __preempt_count_sub(PREEMPT_ACTIVE);
1da177e4 2939
3a5c359a
AK
2940 /*
2941 * Check again in case we missed a preemption opportunity
2942 * between schedule and now.
2943 */
2944 barrier();
5ed0cec0 2945 } while (need_resched());
1da177e4 2946}
376e2424 2947NOKPROBE_SYMBOL(preempt_schedule);
1da177e4 2948EXPORT_SYMBOL(preempt_schedule);
32e475d7 2949#endif /* CONFIG_PREEMPT */
1da177e4
LT
2950
2951/*
2ed6e34f 2952 * this is the entry point to schedule() from kernel preemption
1da177e4
LT
2953 * off of irq context.
2954 * Note, that this is called and return with irqs disabled. This will
2955 * protect us against recursive calling from irq.
2956 */
722a9f92 2957asmlinkage __visible void __sched preempt_schedule_irq(void)
1da177e4 2958{
b22366cd 2959 enum ctx_state prev_state;
6478d880 2960
2ed6e34f 2961 /* Catch callers which need to be fixed */
f27dde8d 2962 BUG_ON(preempt_count() || !irqs_disabled());
1da177e4 2963
b22366cd
FW
2964 prev_state = exception_enter();
2965
3a5c359a 2966 do {
bdb43806 2967 __preempt_count_add(PREEMPT_ACTIVE);
3a5c359a 2968 local_irq_enable();
c259e01a 2969 __schedule();
3a5c359a 2970 local_irq_disable();
bdb43806 2971 __preempt_count_sub(PREEMPT_ACTIVE);
1da177e4 2972
3a5c359a
AK
2973 /*
2974 * Check again in case we missed a preemption opportunity
2975 * between schedule and now.
2976 */
2977 barrier();
5ed0cec0 2978 } while (need_resched());
b22366cd
FW
2979
2980 exception_exit(prev_state);
1da177e4
LT
2981}
2982
63859d4f 2983int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
95cdf3b7 2984 void *key)
1da177e4 2985{
63859d4f 2986 return try_to_wake_up(curr->private, mode, wake_flags);
1da177e4 2987}
1da177e4
LT
2988EXPORT_SYMBOL(default_wake_function);
2989
b29739f9
IM
2990#ifdef CONFIG_RT_MUTEXES
2991
2992/*
2993 * rt_mutex_setprio - set the current priority of a task
2994 * @p: task
2995 * @prio: prio value (kernel-internal form)
2996 *
2997 * This function changes the 'effective' priority of a task. It does
2998 * not touch ->normal_prio like __setscheduler().
2999 *
c365c292
TG
3000 * Used by the rt_mutex code to implement priority inheritance
3001 * logic. Call site only calls if the priority of the task changed.
b29739f9 3002 */
36c8b586 3003void rt_mutex_setprio(struct task_struct *p, int prio)
b29739f9 3004{
da0c1e65 3005 int oldprio, queued, running, enqueue_flag = 0;
70b97a7f 3006 struct rq *rq;
83ab0aa0 3007 const struct sched_class *prev_class;
b29739f9 3008
aab03e05 3009 BUG_ON(prio > MAX_PRIO);
b29739f9 3010
0122ec5b 3011 rq = __task_rq_lock(p);
b29739f9 3012
1c4dd99b
TG
3013 /*
3014 * Idle task boosting is a nono in general. There is one
3015 * exception, when PREEMPT_RT and NOHZ is active:
3016 *
3017 * The idle task calls get_next_timer_interrupt() and holds
3018 * the timer wheel base->lock on the CPU and another CPU wants
3019 * to access the timer (probably to cancel it). We can safely
3020 * ignore the boosting request, as the idle CPU runs this code
3021 * with interrupts disabled and will complete the lock
3022 * protected section without being interrupted. So there is no
3023 * real need to boost.
3024 */
3025 if (unlikely(p == rq->idle)) {
3026 WARN_ON(p != rq->curr);
3027 WARN_ON(p->pi_blocked_on);
3028 goto out_unlock;
3029 }
3030
a8027073 3031 trace_sched_pi_setprio(p, prio);
d5f9f942 3032 oldprio = p->prio;
83ab0aa0 3033 prev_class = p->sched_class;
da0c1e65 3034 queued = task_on_rq_queued(p);
051a1d1a 3035 running = task_current(rq, p);
da0c1e65 3036 if (queued)
69be72c1 3037 dequeue_task(rq, p, 0);
0e1f3483 3038 if (running)
f3cd1c4e 3039 put_prev_task(rq, p);
dd41f596 3040
2d3d891d
DF
3041 /*
3042 * Boosting condition are:
3043 * 1. -rt task is running and holds mutex A
3044 * --> -dl task blocks on mutex A
3045 *
3046 * 2. -dl task is running and holds mutex A
3047 * --> -dl task blocks on mutex A and could preempt the
3048 * running task
3049 */
3050 if (dl_prio(prio)) {
466af29b
ON
3051 struct task_struct *pi_task = rt_mutex_get_top_task(p);
3052 if (!dl_prio(p->normal_prio) ||
3053 (pi_task && dl_entity_preempt(&pi_task->dl, &p->dl))) {
2d3d891d
DF
3054 p->dl.dl_boosted = 1;
3055 p->dl.dl_throttled = 0;
3056 enqueue_flag = ENQUEUE_REPLENISH;
3057 } else
3058 p->dl.dl_boosted = 0;
aab03e05 3059 p->sched_class = &dl_sched_class;
2d3d891d
DF
3060 } else if (rt_prio(prio)) {
3061 if (dl_prio(oldprio))
3062 p->dl.dl_boosted = 0;
3063 if (oldprio < prio)
3064 enqueue_flag = ENQUEUE_HEAD;
dd41f596 3065 p->sched_class = &rt_sched_class;
2d3d891d
DF
3066 } else {
3067 if (dl_prio(oldprio))
3068 p->dl.dl_boosted = 0;
dd41f596 3069 p->sched_class = &fair_sched_class;
2d3d891d 3070 }
dd41f596 3071
b29739f9
IM
3072 p->prio = prio;
3073
0e1f3483
HS
3074 if (running)
3075 p->sched_class->set_curr_task(rq);
da0c1e65 3076 if (queued)
2d3d891d 3077 enqueue_task(rq, p, enqueue_flag);
cb469845 3078
da7a735e 3079 check_class_changed(rq, p, prev_class, oldprio);
1c4dd99b 3080out_unlock:
0122ec5b 3081 __task_rq_unlock(rq);
b29739f9 3082}
b29739f9 3083#endif
d50dde5a 3084
36c8b586 3085void set_user_nice(struct task_struct *p, long nice)
1da177e4 3086{
da0c1e65 3087 int old_prio, delta, queued;
1da177e4 3088 unsigned long flags;
70b97a7f 3089 struct rq *rq;
1da177e4 3090
75e45d51 3091 if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE)
1da177e4
LT
3092 return;
3093 /*
3094 * We have to be careful, if called from sys_setpriority(),
3095 * the task might be in the middle of scheduling on another CPU.
3096 */
3097 rq = task_rq_lock(p, &flags);
3098 /*
3099 * The RT priorities are set via sched_setscheduler(), but we still
3100 * allow the 'normal' nice value to be set - but as expected
3101 * it wont have any effect on scheduling until the task is
aab03e05 3102 * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR:
1da177e4 3103 */
aab03e05 3104 if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
1da177e4
LT
3105 p->static_prio = NICE_TO_PRIO(nice);
3106 goto out_unlock;
3107 }
da0c1e65
KT
3108 queued = task_on_rq_queued(p);
3109 if (queued)
69be72c1 3110 dequeue_task(rq, p, 0);
1da177e4 3111
1da177e4 3112 p->static_prio = NICE_TO_PRIO(nice);
2dd73a4f 3113 set_load_weight(p);
b29739f9
IM
3114 old_prio = p->prio;
3115 p->prio = effective_prio(p);
3116 delta = p->prio - old_prio;
1da177e4 3117
da0c1e65 3118 if (queued) {
371fd7e7 3119 enqueue_task(rq, p, 0);
1da177e4 3120 /*
d5f9f942
AM
3121 * If the task increased its priority or is running and
3122 * lowered its priority, then reschedule its CPU:
1da177e4 3123 */
d5f9f942 3124 if (delta < 0 || (delta > 0 && task_running(rq, p)))
8875125e 3125 resched_curr(rq);
1da177e4
LT
3126 }
3127out_unlock:
0122ec5b 3128 task_rq_unlock(rq, p, &flags);
1da177e4 3129}
1da177e4
LT
3130EXPORT_SYMBOL(set_user_nice);
3131
e43379f1
MM
3132/*
3133 * can_nice - check if a task can reduce its nice value
3134 * @p: task
3135 * @nice: nice value
3136 */
36c8b586 3137int can_nice(const struct task_struct *p, const int nice)
e43379f1 3138{
024f4747 3139 /* convert nice value [19,-20] to rlimit style value [1,40] */
7aa2c016 3140 int nice_rlim = nice_to_rlimit(nice);
48f24c4d 3141
78d7d407 3142 return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
e43379f1
MM
3143 capable(CAP_SYS_NICE));
3144}
3145
1da177e4
LT
3146#ifdef __ARCH_WANT_SYS_NICE
3147
3148/*
3149 * sys_nice - change the priority of the current process.
3150 * @increment: priority increment
3151 *
3152 * sys_setpriority is a more generic, but much slower function that
3153 * does similar things.
3154 */
5add95d4 3155SYSCALL_DEFINE1(nice, int, increment)
1da177e4 3156{
48f24c4d 3157 long nice, retval;
1da177e4
LT
3158
3159 /*
3160 * Setpriority might change our priority at the same moment.
3161 * We don't have to worry. Conceptually one call occurs first
3162 * and we have a single winner.
3163 */
a9467fa3 3164 increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH);
d0ea0268 3165 nice = task_nice(current) + increment;
1da177e4 3166
a9467fa3 3167 nice = clamp_val(nice, MIN_NICE, MAX_NICE);
e43379f1
MM
3168 if (increment < 0 && !can_nice(current, nice))
3169 return -EPERM;
3170
1da177e4
LT
3171 retval = security_task_setnice(current, nice);
3172 if (retval)
3173 return retval;
3174
3175 set_user_nice(current, nice);
3176 return 0;
3177}
3178
3179#endif
3180
3181/**
3182 * task_prio - return the priority value of a given task.
3183 * @p: the task in question.
3184 *
e69f6186 3185 * Return: The priority value as seen by users in /proc.
1da177e4
LT
3186 * RT tasks are offset by -200. Normal tasks are centered
3187 * around 0, value goes from -16 to +15.
3188 */
36c8b586 3189int task_prio(const struct task_struct *p)
1da177e4
LT
3190{
3191 return p->prio - MAX_RT_PRIO;
3192}
3193
1da177e4
LT
3194/**
3195 * idle_cpu - is a given cpu idle currently?
3196 * @cpu: the processor in question.
e69f6186
YB
3197 *
3198 * Return: 1 if the CPU is currently idle. 0 otherwise.
1da177e4
LT
3199 */
3200int idle_cpu(int cpu)
3201{
908a3283
TG
3202 struct rq *rq = cpu_rq(cpu);
3203
3204 if (rq->curr != rq->idle)
3205 return 0;
3206
3207 if (rq->nr_running)
3208 return 0;
3209
3210#ifdef CONFIG_SMP
3211 if (!llist_empty(&rq->wake_list))
3212 return 0;
3213#endif
3214
3215 return 1;
1da177e4
LT
3216}
3217
1da177e4
LT
3218/**
3219 * idle_task - return the idle task for a given cpu.
3220 * @cpu: the processor in question.
e69f6186
YB
3221 *
3222 * Return: The idle task for the cpu @cpu.
1da177e4 3223 */
36c8b586 3224struct task_struct *idle_task(int cpu)
1da177e4
LT
3225{
3226 return cpu_rq(cpu)->idle;
3227}
3228
3229/**
3230 * find_process_by_pid - find a process with a matching PID value.
3231 * @pid: the pid in question.
e69f6186
YB
3232 *
3233 * The task of @pid, if found. %NULL otherwise.
1da177e4 3234 */
a9957449 3235static struct task_struct *find_process_by_pid(pid_t pid)
1da177e4 3236{
228ebcbe 3237 return pid ? find_task_by_vpid(pid) : current;
1da177e4
LT
3238}
3239
aab03e05
DF
3240/*
3241 * This function initializes the sched_dl_entity of a newly becoming
3242 * SCHED_DEADLINE task.
3243 *
3244 * Only the static values are considered here, the actual runtime and the
3245 * absolute deadline will be properly calculated when the task is enqueued
3246 * for the first time with its new policy.
3247 */
3248static void
3249__setparam_dl(struct task_struct *p, const struct sched_attr *attr)
3250{
3251 struct sched_dl_entity *dl_se = &p->dl;
3252
3253 init_dl_task_timer(dl_se);
3254 dl_se->dl_runtime = attr->sched_runtime;
3255 dl_se->dl_deadline = attr->sched_deadline;
755378a4 3256 dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline;
aab03e05 3257 dl_se->flags = attr->sched_flags;
332ac17e 3258 dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime);
aab03e05
DF
3259 dl_se->dl_throttled = 0;
3260 dl_se->dl_new = 1;
5bfd126e 3261 dl_se->dl_yielded = 0;
aab03e05
DF
3262}
3263
c13db6b1
SR
3264/*
3265 * sched_setparam() passes in -1 for its policy, to let the functions
3266 * it calls know not to change it.
3267 */
3268#define SETPARAM_POLICY -1
3269
c365c292
TG
3270static void __setscheduler_params(struct task_struct *p,
3271 const struct sched_attr *attr)
1da177e4 3272{
d50dde5a
DF
3273 int policy = attr->sched_policy;
3274
c13db6b1 3275 if (policy == SETPARAM_POLICY)
39fd8fd2
PZ
3276 policy = p->policy;
3277
1da177e4 3278 p->policy = policy;
d50dde5a 3279
aab03e05
DF
3280 if (dl_policy(policy))
3281 __setparam_dl(p, attr);
39fd8fd2 3282 else if (fair_policy(policy))
d50dde5a
DF
3283 p->static_prio = NICE_TO_PRIO(attr->sched_nice);
3284
39fd8fd2
PZ
3285 /*
3286 * __sched_setscheduler() ensures attr->sched_priority == 0 when
3287 * !rt_policy. Always setting this ensures that things like
3288 * getparam()/getattr() don't report silly values for !rt tasks.
3289 */
3290 p->rt_priority = attr->sched_priority;
383afd09 3291 p->normal_prio = normal_prio(p);
c365c292
TG
3292 set_load_weight(p);
3293}
39fd8fd2 3294
c365c292
TG
3295/* Actually do priority change: must hold pi & rq lock. */
3296static void __setscheduler(struct rq *rq, struct task_struct *p,
3297 const struct sched_attr *attr)
3298{
3299 __setscheduler_params(p, attr);
d50dde5a 3300
383afd09
SR
3301 /*
3302 * If we get here, there was no pi waiters boosting the
3303 * task. It is safe to use the normal prio.
3304 */
3305 p->prio = normal_prio(p);
3306
aab03e05
DF
3307 if (dl_prio(p->prio))
3308 p->sched_class = &dl_sched_class;
3309 else if (rt_prio(p->prio))
ffd44db5
PZ
3310 p->sched_class = &rt_sched_class;
3311 else
3312 p->sched_class = &fair_sched_class;
1da177e4 3313}
aab03e05
DF
3314
3315static void
3316__getparam_dl(struct task_struct *p, struct sched_attr *attr)
3317{
3318 struct sched_dl_entity *dl_se = &p->dl;
3319
3320 attr->sched_priority = p->rt_priority;
3321 attr->sched_runtime = dl_se->dl_runtime;
3322 attr->sched_deadline = dl_se->dl_deadline;
755378a4 3323 attr->sched_period = dl_se->dl_period;
aab03e05
DF
3324 attr->sched_flags = dl_se->flags;
3325}
3326
3327/*
3328 * This function validates the new parameters of a -deadline task.
3329 * We ask for the deadline not being zero, and greater or equal
755378a4 3330 * than the runtime, as well as the period of being zero or
332ac17e 3331 * greater than deadline. Furthermore, we have to be sure that
b0827819
JL
3332 * user parameters are above the internal resolution of 1us (we
3333 * check sched_runtime only since it is always the smaller one) and
3334 * below 2^63 ns (we have to check both sched_deadline and
3335 * sched_period, as the latter can be zero).
aab03e05
DF
3336 */
3337static bool
3338__checkparam_dl(const struct sched_attr *attr)
3339{
b0827819
JL
3340 /* deadline != 0 */
3341 if (attr->sched_deadline == 0)
3342 return false;
3343
3344 /*
3345 * Since we truncate DL_SCALE bits, make sure we're at least
3346 * that big.
3347 */
3348 if (attr->sched_runtime < (1ULL << DL_SCALE))
3349 return false;
3350
3351 /*
3352 * Since we use the MSB for wrap-around and sign issues, make
3353 * sure it's not set (mind that period can be equal to zero).
3354 */
3355 if (attr->sched_deadline & (1ULL << 63) ||
3356 attr->sched_period & (1ULL << 63))
3357 return false;
3358
3359 /* runtime <= deadline <= period (if period != 0) */
3360 if ((attr->sched_period != 0 &&
3361 attr->sched_period < attr->sched_deadline) ||
3362 attr->sched_deadline < attr->sched_runtime)
3363 return false;
3364
3365 return true;
aab03e05
DF
3366}
3367
c69e8d9c
DH
3368/*
3369 * check the target process has a UID that matches the current process's
3370 */
3371static bool check_same_owner(struct task_struct *p)
3372{
3373 const struct cred *cred = current_cred(), *pcred;
3374 bool match;
3375
3376 rcu_read_lock();
3377 pcred = __task_cred(p);
9c806aa0
EB
3378 match = (uid_eq(cred->euid, pcred->euid) ||
3379 uid_eq(cred->euid, pcred->uid));
c69e8d9c
DH
3380 rcu_read_unlock();
3381 return match;
3382}
3383
d50dde5a
DF
3384static int __sched_setscheduler(struct task_struct *p,
3385 const struct sched_attr *attr,
3386 bool user)
1da177e4 3387{
383afd09
SR
3388 int newprio = dl_policy(attr->sched_policy) ? MAX_DL_PRIO - 1 :
3389 MAX_RT_PRIO - 1 - attr->sched_priority;
da0c1e65 3390 int retval, oldprio, oldpolicy = -1, queued, running;
d50dde5a 3391 int policy = attr->sched_policy;
1da177e4 3392 unsigned long flags;
83ab0aa0 3393 const struct sched_class *prev_class;
70b97a7f 3394 struct rq *rq;
ca94c442 3395 int reset_on_fork;
1da177e4 3396
66e5393a
SR
3397 /* may grab non-irq protected spin_locks */
3398 BUG_ON(in_interrupt());
1da177e4
LT
3399recheck:
3400 /* double check policy once rq lock held */
ca94c442
LP
3401 if (policy < 0) {
3402 reset_on_fork = p->sched_reset_on_fork;
1da177e4 3403 policy = oldpolicy = p->policy;
ca94c442 3404 } else {
7479f3c9 3405 reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK);
ca94c442 3406
aab03e05
DF
3407 if (policy != SCHED_DEADLINE &&
3408 policy != SCHED_FIFO && policy != SCHED_RR &&
ca94c442
LP
3409 policy != SCHED_NORMAL && policy != SCHED_BATCH &&
3410 policy != SCHED_IDLE)
3411 return -EINVAL;
3412 }
3413
7479f3c9
PZ
3414 if (attr->sched_flags & ~(SCHED_FLAG_RESET_ON_FORK))
3415 return -EINVAL;
3416
1da177e4
LT
3417 /*
3418 * Valid priorities for SCHED_FIFO and SCHED_RR are
dd41f596
IM
3419 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
3420 * SCHED_BATCH and SCHED_IDLE is 0.
1da177e4 3421 */
0bb040a4 3422 if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) ||
d50dde5a 3423 (!p->mm && attr->sched_priority > MAX_RT_PRIO-1))
1da177e4 3424 return -EINVAL;
aab03e05
DF
3425 if ((dl_policy(policy) && !__checkparam_dl(attr)) ||
3426 (rt_policy(policy) != (attr->sched_priority != 0)))
1da177e4
LT
3427 return -EINVAL;
3428
37e4ab3f
OC
3429 /*
3430 * Allow unprivileged RT tasks to decrease priority:
3431 */
961ccddd 3432 if (user && !capable(CAP_SYS_NICE)) {
d50dde5a 3433 if (fair_policy(policy)) {
d0ea0268 3434 if (attr->sched_nice < task_nice(p) &&
eaad4513 3435 !can_nice(p, attr->sched_nice))
d50dde5a
DF
3436 return -EPERM;
3437 }
3438
e05606d3 3439 if (rt_policy(policy)) {
a44702e8
ON
3440 unsigned long rlim_rtprio =
3441 task_rlimit(p, RLIMIT_RTPRIO);
8dc3e909
ON
3442
3443 /* can't set/change the rt policy */
3444 if (policy != p->policy && !rlim_rtprio)
3445 return -EPERM;
3446
3447 /* can't increase priority */
d50dde5a
DF
3448 if (attr->sched_priority > p->rt_priority &&
3449 attr->sched_priority > rlim_rtprio)
8dc3e909
ON
3450 return -EPERM;
3451 }
c02aa73b 3452
d44753b8
JL
3453 /*
3454 * Can't set/change SCHED_DEADLINE policy at all for now
3455 * (safest behavior); in the future we would like to allow
3456 * unprivileged DL tasks to increase their relative deadline
3457 * or reduce their runtime (both ways reducing utilization)
3458 */
3459 if (dl_policy(policy))
3460 return -EPERM;
3461
dd41f596 3462 /*
c02aa73b
DH
3463 * Treat SCHED_IDLE as nice 20. Only allow a switch to
3464 * SCHED_NORMAL if the RLIMIT_NICE would normally permit it.
dd41f596 3465 */
c02aa73b 3466 if (p->policy == SCHED_IDLE && policy != SCHED_IDLE) {
d0ea0268 3467 if (!can_nice(p, task_nice(p)))
c02aa73b
DH
3468 return -EPERM;
3469 }
5fe1d75f 3470
37e4ab3f 3471 /* can't change other user's priorities */
c69e8d9c 3472 if (!check_same_owner(p))
37e4ab3f 3473 return -EPERM;
ca94c442
LP
3474
3475 /* Normal users shall not reset the sched_reset_on_fork flag */
3476 if (p->sched_reset_on_fork && !reset_on_fork)
3477 return -EPERM;
37e4ab3f 3478 }
1da177e4 3479
725aad24 3480 if (user) {
b0ae1981 3481 retval = security_task_setscheduler(p);
725aad24
JF
3482 if (retval)
3483 return retval;
3484 }
3485
b29739f9
IM
3486 /*
3487 * make sure no PI-waiters arrive (or leave) while we are
3488 * changing the priority of the task:
0122ec5b 3489 *
25985edc 3490 * To be able to change p->policy safely, the appropriate
1da177e4
LT
3491 * runqueue lock must be held.
3492 */
0122ec5b 3493 rq = task_rq_lock(p, &flags);
dc61b1d6 3494
34f971f6
PZ
3495 /*
3496 * Changing the policy of the stop threads its a very bad idea
3497 */
3498 if (p == rq->stop) {
0122ec5b 3499 task_rq_unlock(rq, p, &flags);
34f971f6
PZ
3500 return -EINVAL;
3501 }
3502
a51e9198 3503 /*
d6b1e911
TG
3504 * If not changing anything there's no need to proceed further,
3505 * but store a possible modification of reset_on_fork.
a51e9198 3506 */
d50dde5a 3507 if (unlikely(policy == p->policy)) {
d0ea0268 3508 if (fair_policy(policy) && attr->sched_nice != task_nice(p))
d50dde5a
DF
3509 goto change;
3510 if (rt_policy(policy) && attr->sched_priority != p->rt_priority)
3511 goto change;
aab03e05
DF
3512 if (dl_policy(policy))
3513 goto change;
d50dde5a 3514
d6b1e911 3515 p->sched_reset_on_fork = reset_on_fork;
45afb173 3516 task_rq_unlock(rq, p, &flags);
a51e9198
DF
3517 return 0;
3518 }
d50dde5a 3519change:
a51e9198 3520
dc61b1d6 3521 if (user) {
332ac17e 3522#ifdef CONFIG_RT_GROUP_SCHED
dc61b1d6
PZ
3523 /*
3524 * Do not allow realtime tasks into groups that have no runtime
3525 * assigned.
3526 */
3527 if (rt_bandwidth_enabled() && rt_policy(policy) &&
f4493771
MG
3528 task_group(p)->rt_bandwidth.rt_runtime == 0 &&
3529 !task_group_is_autogroup(task_group(p))) {
0122ec5b 3530 task_rq_unlock(rq, p, &flags);
dc61b1d6
PZ
3531 return -EPERM;
3532 }
dc61b1d6 3533#endif
332ac17e
DF
3534#ifdef CONFIG_SMP
3535 if (dl_bandwidth_enabled() && dl_policy(policy)) {
3536 cpumask_t *span = rq->rd->span;
332ac17e
DF
3537
3538 /*
3539 * Don't allow tasks with an affinity mask smaller than
3540 * the entire root_domain to become SCHED_DEADLINE. We
3541 * will also fail if there's no bandwidth available.
3542 */
e4099a5e
PZ
3543 if (!cpumask_subset(span, &p->cpus_allowed) ||
3544 rq->rd->dl_bw.bw == 0) {
332ac17e
DF
3545 task_rq_unlock(rq, p, &flags);
3546 return -EPERM;
3547 }
3548 }
3549#endif
3550 }
dc61b1d6 3551
1da177e4
LT
3552 /* recheck policy now with rq lock held */
3553 if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
3554 policy = oldpolicy = -1;
0122ec5b 3555 task_rq_unlock(rq, p, &flags);
1da177e4
LT
3556 goto recheck;
3557 }
332ac17e
DF
3558
3559 /*
3560 * If setscheduling to SCHED_DEADLINE (or changing the parameters
3561 * of a SCHED_DEADLINE task) we need to check if enough bandwidth
3562 * is available.
3563 */
e4099a5e 3564 if ((dl_policy(policy) || dl_task(p)) && dl_overflow(p, policy, attr)) {
332ac17e
DF
3565 task_rq_unlock(rq, p, &flags);
3566 return -EBUSY;
3567 }
3568
c365c292
TG
3569 p->sched_reset_on_fork = reset_on_fork;
3570 oldprio = p->prio;
3571
3572 /*
3573 * Special case for priority boosted tasks.
3574 *
3575 * If the new priority is lower or equal (user space view)
3576 * than the current (boosted) priority, we just store the new
3577 * normal parameters and do not touch the scheduler class and
3578 * the runqueue. This will be done when the task deboost
3579 * itself.
3580 */
3581 if (rt_mutex_check_prio(p, newprio)) {
3582 __setscheduler_params(p, attr);
3583 task_rq_unlock(rq, p, &flags);
3584 return 0;
3585 }
3586
da0c1e65 3587 queued = task_on_rq_queued(p);
051a1d1a 3588 running = task_current(rq, p);
da0c1e65 3589 if (queued)
4ca9b72b 3590 dequeue_task(rq, p, 0);
0e1f3483 3591 if (running)
f3cd1c4e 3592 put_prev_task(rq, p);
f6b53205 3593
83ab0aa0 3594 prev_class = p->sched_class;
d50dde5a 3595 __setscheduler(rq, p, attr);
f6b53205 3596
0e1f3483
HS
3597 if (running)
3598 p->sched_class->set_curr_task(rq);
da0c1e65 3599 if (queued) {
81a44c54
TG
3600 /*
3601 * We enqueue to tail when the priority of a task is
3602 * increased (user space view).
3603 */
3604 enqueue_task(rq, p, oldprio <= p->prio ? ENQUEUE_HEAD : 0);
3605 }
cb469845 3606
da7a735e 3607 check_class_changed(rq, p, prev_class, oldprio);
0122ec5b 3608 task_rq_unlock(rq, p, &flags);
b29739f9 3609
95e02ca9
TG
3610 rt_mutex_adjust_pi(p);
3611
1da177e4
LT
3612 return 0;
3613}
961ccddd 3614
7479f3c9
PZ
3615static int _sched_setscheduler(struct task_struct *p, int policy,
3616 const struct sched_param *param, bool check)
3617{
3618 struct sched_attr attr = {
3619 .sched_policy = policy,
3620 .sched_priority = param->sched_priority,
3621 .sched_nice = PRIO_TO_NICE(p->static_prio),
3622 };
3623
c13db6b1
SR
3624 /* Fixup the legacy SCHED_RESET_ON_FORK hack. */
3625 if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) {
7479f3c9
PZ
3626 attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
3627 policy &= ~SCHED_RESET_ON_FORK;
3628 attr.sched_policy = policy;
3629 }
3630
3631 return __sched_setscheduler(p, &attr, check);
3632}
961ccddd
RR
3633/**
3634 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
3635 * @p: the task in question.
3636 * @policy: new policy.
3637 * @param: structure containing the new RT priority.
3638 *
e69f6186
YB
3639 * Return: 0 on success. An error code otherwise.
3640 *
961ccddd
RR
3641 * NOTE that the task may be already dead.
3642 */
3643int sched_setscheduler(struct task_struct *p, int policy,
fe7de49f 3644 const struct sched_param *param)
961ccddd 3645{
7479f3c9 3646 return _sched_setscheduler(p, policy, param, true);
961ccddd 3647}
1da177e4
LT
3648EXPORT_SYMBOL_GPL(sched_setscheduler);
3649
d50dde5a
DF
3650int sched_setattr(struct task_struct *p, const struct sched_attr *attr)
3651{
3652 return __sched_setscheduler(p, attr, true);
3653}
3654EXPORT_SYMBOL_GPL(sched_setattr);
3655
961ccddd
RR
3656/**
3657 * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
3658 * @p: the task in question.
3659 * @policy: new policy.
3660 * @param: structure containing the new RT priority.
3661 *
3662 * Just like sched_setscheduler, only don't bother checking if the
3663 * current context has permission. For example, this is needed in
3664 * stop_machine(): we create temporary high priority worker threads,
3665 * but our caller might not have that capability.
e69f6186
YB
3666 *
3667 * Return: 0 on success. An error code otherwise.
961ccddd
RR
3668 */
3669int sched_setscheduler_nocheck(struct task_struct *p, int policy,
fe7de49f 3670 const struct sched_param *param)
961ccddd 3671{
7479f3c9 3672 return _sched_setscheduler(p, policy, param, false);
961ccddd
RR
3673}
3674
95cdf3b7
IM
3675static int
3676do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
1da177e4 3677{
1da177e4
LT
3678 struct sched_param lparam;
3679 struct task_struct *p;
36c8b586 3680 int retval;
1da177e4
LT
3681
3682 if (!param || pid < 0)
3683 return -EINVAL;
3684 if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
3685 return -EFAULT;
5fe1d75f
ON
3686
3687 rcu_read_lock();
3688 retval = -ESRCH;
1da177e4 3689 p = find_process_by_pid(pid);
5fe1d75f
ON
3690 if (p != NULL)
3691 retval = sched_setscheduler(p, policy, &lparam);
3692 rcu_read_unlock();
36c8b586 3693
1da177e4
LT
3694 return retval;
3695}
3696
d50dde5a
DF
3697/*
3698 * Mimics kernel/events/core.c perf_copy_attr().
3699 */
3700static int sched_copy_attr(struct sched_attr __user *uattr,
3701 struct sched_attr *attr)
3702{
3703 u32 size;
3704 int ret;
3705
3706 if (!access_ok(VERIFY_WRITE, uattr, SCHED_ATTR_SIZE_VER0))
3707 return -EFAULT;
3708
3709 /*
3710 * zero the full structure, so that a short copy will be nice.
3711 */
3712 memset(attr, 0, sizeof(*attr));
3713
3714 ret = get_user(size, &uattr->size);
3715 if (ret)
3716 return ret;
3717
3718 if (size > PAGE_SIZE) /* silly large */
3719 goto err_size;
3720
3721 if (!size) /* abi compat */
3722 size = SCHED_ATTR_SIZE_VER0;
3723
3724 if (size < SCHED_ATTR_SIZE_VER0)
3725 goto err_size;
3726
3727 /*
3728 * If we're handed a bigger struct than we know of,
3729 * ensure all the unknown bits are 0 - i.e. new
3730 * user-space does not rely on any kernel feature
3731 * extensions we dont know about yet.
3732 */
3733 if (size > sizeof(*attr)) {
3734 unsigned char __user *addr;
3735 unsigned char __user *end;
3736 unsigned char val;
3737
3738 addr = (void __user *)uattr + sizeof(*attr);
3739 end = (void __user *)uattr + size;
3740
3741 for (; addr < end; addr++) {
3742 ret = get_user(val, addr);
3743 if (ret)
3744 return ret;
3745 if (val)
3746 goto err_size;
3747 }
3748 size = sizeof(*attr);
3749 }
3750
3751 ret = copy_from_user(attr, uattr, size);
3752 if (ret)
3753 return -EFAULT;
3754
3755 /*
3756 * XXX: do we want to be lenient like existing syscalls; or do we want
3757 * to be strict and return an error on out-of-bounds values?
3758 */
75e45d51 3759 attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE);
d50dde5a 3760
e78c7bca 3761 return 0;
d50dde5a
DF
3762
3763err_size:
3764 put_user(sizeof(*attr), &uattr->size);
e78c7bca 3765 return -E2BIG;
d50dde5a
DF
3766}
3767
1da177e4
LT
3768/**
3769 * sys_sched_setscheduler - set/change the scheduler policy and RT priority
3770 * @pid: the pid in question.
3771 * @policy: new policy.
3772 * @param: structure containing the new RT priority.
e69f6186
YB
3773 *
3774 * Return: 0 on success. An error code otherwise.
1da177e4 3775 */
5add95d4
HC
3776SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy,
3777 struct sched_param __user *, param)
1da177e4 3778{
c21761f1
JB
3779 /* negative values for policy are not valid */
3780 if (policy < 0)
3781 return -EINVAL;
3782
1da177e4
LT
3783 return do_sched_setscheduler(pid, policy, param);
3784}
3785
3786/**
3787 * sys_sched_setparam - set/change the RT priority of a thread
3788 * @pid: the pid in question.
3789 * @param: structure containing the new RT priority.
e69f6186
YB
3790 *
3791 * Return: 0 on success. An error code otherwise.
1da177e4 3792 */
5add95d4 3793SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
1da177e4 3794{
c13db6b1 3795 return do_sched_setscheduler(pid, SETPARAM_POLICY, param);
1da177e4
LT
3796}
3797
d50dde5a
DF
3798/**
3799 * sys_sched_setattr - same as above, but with extended sched_attr
3800 * @pid: the pid in question.
5778fccf 3801 * @uattr: structure containing the extended parameters.
db66d756 3802 * @flags: for future extension.
d50dde5a 3803 */
6d35ab48
PZ
3804SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr,
3805 unsigned int, flags)
d50dde5a
DF
3806{
3807 struct sched_attr attr;
3808 struct task_struct *p;
3809 int retval;
3810
6d35ab48 3811 if (!uattr || pid < 0 || flags)
d50dde5a
DF
3812 return -EINVAL;
3813
143cf23d
MK
3814 retval = sched_copy_attr(uattr, &attr);
3815 if (retval)
3816 return retval;
d50dde5a 3817
b14ed2c2 3818 if ((int)attr.sched_policy < 0)
dbdb2275 3819 return -EINVAL;
d50dde5a
DF
3820
3821 rcu_read_lock();
3822 retval = -ESRCH;
3823 p = find_process_by_pid(pid);
3824 if (p != NULL)
3825 retval = sched_setattr(p, &attr);
3826 rcu_read_unlock();
3827
3828 return retval;
3829}
3830
1da177e4
LT
3831/**
3832 * sys_sched_getscheduler - get the policy (scheduling class) of a thread
3833 * @pid: the pid in question.
e69f6186
YB
3834 *
3835 * Return: On success, the policy of the thread. Otherwise, a negative error
3836 * code.
1da177e4 3837 */
5add95d4 3838SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
1da177e4 3839{
36c8b586 3840 struct task_struct *p;
3a5c359a 3841 int retval;
1da177e4
LT
3842
3843 if (pid < 0)
3a5c359a 3844 return -EINVAL;
1da177e4
LT
3845
3846 retval = -ESRCH;
5fe85be0 3847 rcu_read_lock();
1da177e4
LT
3848 p = find_process_by_pid(pid);
3849 if (p) {
3850 retval = security_task_getscheduler(p);
3851 if (!retval)
ca94c442
LP
3852 retval = p->policy
3853 | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
1da177e4 3854 }
5fe85be0 3855 rcu_read_unlock();
1da177e4
LT
3856 return retval;
3857}
3858
3859/**
ca94c442 3860 * sys_sched_getparam - get the RT priority of a thread
1da177e4
LT
3861 * @pid: the pid in question.
3862 * @param: structure containing the RT priority.
e69f6186
YB
3863 *
3864 * Return: On success, 0 and the RT priority is in @param. Otherwise, an error
3865 * code.
1da177e4 3866 */
5add95d4 3867SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
1da177e4 3868{
ce5f7f82 3869 struct sched_param lp = { .sched_priority = 0 };
36c8b586 3870 struct task_struct *p;
3a5c359a 3871 int retval;
1da177e4
LT
3872
3873 if (!param || pid < 0)
3a5c359a 3874 return -EINVAL;
1da177e4 3875
5fe85be0 3876 rcu_read_lock();
1da177e4
LT
3877 p = find_process_by_pid(pid);
3878 retval = -ESRCH;
3879 if (!p)
3880 goto out_unlock;
3881
3882 retval = security_task_getscheduler(p);
3883 if (retval)
3884 goto out_unlock;
3885
ce5f7f82
PZ
3886 if (task_has_rt_policy(p))
3887 lp.sched_priority = p->rt_priority;
5fe85be0 3888 rcu_read_unlock();
1da177e4
LT
3889
3890 /*
3891 * This one might sleep, we cannot do it with a spinlock held ...
3892 */
3893 retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;
3894
1da177e4
LT
3895 return retval;
3896
3897out_unlock:
5fe85be0 3898 rcu_read_unlock();
1da177e4
LT
3899 return retval;
3900}
3901
d50dde5a
DF
3902static int sched_read_attr(struct sched_attr __user *uattr,
3903 struct sched_attr *attr,
3904 unsigned int usize)
3905{
3906 int ret;
3907
3908 if (!access_ok(VERIFY_WRITE, uattr, usize))
3909 return -EFAULT;
3910
3911 /*
3912 * If we're handed a smaller struct than we know of,
3913 * ensure all the unknown bits are 0 - i.e. old
3914 * user-space does not get uncomplete information.
3915 */
3916 if (usize < sizeof(*attr)) {
3917 unsigned char *addr;
3918 unsigned char *end;
3919
3920 addr = (void *)attr + usize;
3921 end = (void *)attr + sizeof(*attr);
3922
3923 for (; addr < end; addr++) {
3924 if (*addr)
22400674 3925 return -EFBIG;
d50dde5a
DF
3926 }
3927
3928 attr->size = usize;
3929 }
3930
4efbc454 3931 ret = copy_to_user(uattr, attr, attr->size);
d50dde5a
DF
3932 if (ret)
3933 return -EFAULT;
3934
22400674 3935 return 0;
d50dde5a
DF
3936}
3937
3938/**
aab03e05 3939 * sys_sched_getattr - similar to sched_getparam, but with sched_attr
d50dde5a 3940 * @pid: the pid in question.
5778fccf 3941 * @uattr: structure containing the extended parameters.
d50dde5a 3942 * @size: sizeof(attr) for fwd/bwd comp.
db66d756 3943 * @flags: for future extension.
d50dde5a 3944 */
6d35ab48
PZ
3945SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr,
3946 unsigned int, size, unsigned int, flags)
d50dde5a
DF
3947{
3948 struct sched_attr attr = {
3949 .size = sizeof(struct sched_attr),
3950 };
3951 struct task_struct *p;
3952 int retval;
3953
3954 if (!uattr || pid < 0 || size > PAGE_SIZE ||
6d35ab48 3955 size < SCHED_ATTR_SIZE_VER0 || flags)
d50dde5a
DF
3956 return -EINVAL;
3957
3958 rcu_read_lock();
3959 p = find_process_by_pid(pid);
3960 retval = -ESRCH;
3961 if (!p)
3962 goto out_unlock;
3963
3964 retval = security_task_getscheduler(p);
3965 if (retval)
3966 goto out_unlock;
3967
3968 attr.sched_policy = p->policy;
7479f3c9
PZ
3969 if (p->sched_reset_on_fork)
3970 attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
aab03e05
DF
3971 if (task_has_dl_policy(p))
3972 __getparam_dl(p, &attr);
3973 else if (task_has_rt_policy(p))
d50dde5a
DF
3974 attr.sched_priority = p->rt_priority;
3975 else
d0ea0268 3976 attr.sched_nice = task_nice(p);
d50dde5a
DF
3977
3978 rcu_read_unlock();
3979
3980 retval = sched_read_attr(uattr, &attr, size);
3981 return retval;
3982
3983out_unlock:
3984 rcu_read_unlock();
3985 return retval;
3986}
3987
96f874e2 3988long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
1da177e4 3989{
5a16f3d3 3990 cpumask_var_t cpus_allowed, new_mask;
36c8b586
IM
3991 struct task_struct *p;
3992 int retval;
1da177e4 3993
23f5d142 3994 rcu_read_lock();
1da177e4
LT
3995
3996 p = find_process_by_pid(pid);
3997 if (!p) {
23f5d142 3998 rcu_read_unlock();
1da177e4
LT
3999 return -ESRCH;
4000 }
4001
23f5d142 4002 /* Prevent p going away */
1da177e4 4003 get_task_struct(p);
23f5d142 4004 rcu_read_unlock();
1da177e4 4005
14a40ffc
TH
4006 if (p->flags & PF_NO_SETAFFINITY) {
4007 retval = -EINVAL;
4008 goto out_put_task;
4009 }
5a16f3d3
RR
4010 if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
4011 retval = -ENOMEM;
4012 goto out_put_task;
4013 }
4014 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
4015 retval = -ENOMEM;
4016 goto out_free_cpus_allowed;
4017 }
1da177e4 4018 retval = -EPERM;
4c44aaaf
EB
4019 if (!check_same_owner(p)) {
4020 rcu_read_lock();
4021 if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) {
4022 rcu_read_unlock();
4023 goto out_unlock;
4024 }
4025 rcu_read_unlock();
4026 }
1da177e4 4027
b0ae1981 4028 retval = security_task_setscheduler(p);
e7834f8f
DQ
4029 if (retval)
4030 goto out_unlock;
4031
e4099a5e
PZ
4032
4033 cpuset_cpus_allowed(p, cpus_allowed);
4034 cpumask_and(new_mask, in_mask, cpus_allowed);
4035
332ac17e
DF
4036 /*
4037 * Since bandwidth control happens on root_domain basis,
4038 * if admission test is enabled, we only admit -deadline
4039 * tasks allowed to run on all the CPUs in the task's
4040 * root_domain.
4041 */
4042#ifdef CONFIG_SMP
4043 if (task_has_dl_policy(p)) {
4044 const struct cpumask *span = task_rq(p)->rd->span;
4045
e4099a5e 4046 if (dl_bandwidth_enabled() && !cpumask_subset(span, new_mask)) {
332ac17e
DF
4047 retval = -EBUSY;
4048 goto out_unlock;
4049 }
4050 }
4051#endif
49246274 4052again:
5a16f3d3 4053 retval = set_cpus_allowed_ptr(p, new_mask);
1da177e4 4054
8707d8b8 4055 if (!retval) {
5a16f3d3
RR
4056 cpuset_cpus_allowed(p, cpus_allowed);
4057 if (!cpumask_subset(new_mask, cpus_allowed)) {
8707d8b8
PM
4058 /*
4059 * We must have raced with a concurrent cpuset
4060 * update. Just reset the cpus_allowed to the
4061 * cpuset's cpus_allowed
4062 */
5a16f3d3 4063 cpumask_copy(new_mask, cpus_allowed);
8707d8b8
PM
4064 goto again;
4065 }
4066 }
1da177e4 4067out_unlock:
5a16f3d3
RR
4068 free_cpumask_var(new_mask);
4069out_free_cpus_allowed:
4070 free_cpumask_var(cpus_allowed);
4071out_put_task:
1da177e4 4072 put_task_struct(p);
1da177e4
LT
4073 return retval;
4074}
4075
4076static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
96f874e2 4077 struct cpumask *new_mask)
1da177e4 4078{
96f874e2
RR
4079 if (len < cpumask_size())
4080 cpumask_clear(new_mask);
4081 else if (len > cpumask_size())
4082 len = cpumask_size();
4083
1da177e4
LT
4084 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
4085}
4086
4087/**
4088 * sys_sched_setaffinity - set the cpu affinity of a process
4089 * @pid: pid of the process
4090 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
4091 * @user_mask_ptr: user-space pointer to the new cpu mask
e69f6186
YB
4092 *
4093 * Return: 0 on success. An error code otherwise.
1da177e4 4094 */
5add95d4
HC
4095SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
4096 unsigned long __user *, user_mask_ptr)
1da177e4 4097{
5a16f3d3 4098 cpumask_var_t new_mask;
1da177e4
LT
4099 int retval;
4100
5a16f3d3
RR
4101 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
4102 return -ENOMEM;
1da177e4 4103
5a16f3d3
RR
4104 retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
4105 if (retval == 0)
4106 retval = sched_setaffinity(pid, new_mask);
4107 free_cpumask_var(new_mask);
4108 return retval;
1da177e4
LT
4109}
4110
96f874e2 4111long sched_getaffinity(pid_t pid, struct cpumask *mask)
1da177e4 4112{
36c8b586 4113 struct task_struct *p;
31605683 4114 unsigned long flags;
1da177e4 4115 int retval;
1da177e4 4116
23f5d142 4117 rcu_read_lock();
1da177e4
LT
4118
4119 retval = -ESRCH;
4120 p = find_process_by_pid(pid);
4121 if (!p)
4122 goto out_unlock;
4123
e7834f8f
DQ
4124 retval = security_task_getscheduler(p);
4125 if (retval)
4126 goto out_unlock;
4127
013fdb80 4128 raw_spin_lock_irqsave(&p->pi_lock, flags);
6acce3ef 4129 cpumask_and(mask, &p->cpus_allowed, cpu_active_mask);
013fdb80 4130 raw_spin_unlock_irqrestore(&p->pi_lock, flags);
1da177e4
LT
4131
4132out_unlock:
23f5d142 4133 rcu_read_unlock();
1da177e4 4134
9531b62f 4135 return retval;
1da177e4
LT
4136}
4137
4138/**
4139 * sys_sched_getaffinity - get the cpu affinity of a process
4140 * @pid: pid of the process
4141 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
4142 * @user_mask_ptr: user-space pointer to hold the current cpu mask
e69f6186
YB
4143 *
4144 * Return: 0 on success. An error code otherwise.
1da177e4 4145 */
5add95d4
HC
4146SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
4147 unsigned long __user *, user_mask_ptr)
1da177e4
LT
4148{
4149 int ret;
f17c8607 4150 cpumask_var_t mask;
1da177e4 4151
84fba5ec 4152 if ((len * BITS_PER_BYTE) < nr_cpu_ids)
cd3d8031
KM
4153 return -EINVAL;
4154 if (len & (sizeof(unsigned long)-1))
1da177e4
LT
4155 return -EINVAL;
4156
f17c8607
RR
4157 if (!alloc_cpumask_var(&mask, GFP_KERNEL))
4158 return -ENOMEM;
1da177e4 4159
f17c8607
RR
4160 ret = sched_getaffinity(pid, mask);
4161 if (ret == 0) {
8bc037fb 4162 size_t retlen = min_t(size_t, len, cpumask_size());
cd3d8031
KM
4163
4164 if (copy_to_user(user_mask_ptr, mask, retlen))
f17c8607
RR
4165 ret = -EFAULT;
4166 else
cd3d8031 4167 ret = retlen;
f17c8607
RR
4168 }
4169 free_cpumask_var(mask);
1da177e4 4170
f17c8607 4171 return ret;
1da177e4
LT
4172}
4173
4174/**
4175 * sys_sched_yield - yield the current processor to other threads.
4176 *
dd41f596
IM
4177 * This function yields the current CPU to other tasks. If there are no
4178 * other threads running on this CPU then this function will return.
e69f6186
YB
4179 *
4180 * Return: 0.
1da177e4 4181 */
5add95d4 4182SYSCALL_DEFINE0(sched_yield)
1da177e4 4183{
70b97a7f 4184 struct rq *rq = this_rq_lock();
1da177e4 4185
2d72376b 4186 schedstat_inc(rq, yld_count);
4530d7ab 4187 current->sched_class->yield_task(rq);
1da177e4
LT
4188
4189 /*
4190 * Since we are going to call schedule() anyway, there's
4191 * no need to preempt or enable interrupts:
4192 */
4193 __release(rq->lock);
8a25d5de 4194 spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
9828ea9d 4195 do_raw_spin_unlock(&rq->lock);
ba74c144 4196 sched_preempt_enable_no_resched();
1da177e4
LT
4197
4198 schedule();
4199
4200 return 0;
4201}
4202
e7b38404 4203static void __cond_resched(void)
1da177e4 4204{
bdb43806 4205 __preempt_count_add(PREEMPT_ACTIVE);
c259e01a 4206 __schedule();
bdb43806 4207 __preempt_count_sub(PREEMPT_ACTIVE);
1da177e4
LT
4208}
4209
02b67cc3 4210int __sched _cond_resched(void)
1da177e4 4211{
d86ee480 4212 if (should_resched()) {
1da177e4
LT
4213 __cond_resched();
4214 return 1;
4215 }
4216 return 0;
4217}
02b67cc3 4218EXPORT_SYMBOL(_cond_resched);
1da177e4
LT
4219
4220/*
613afbf8 4221 * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
1da177e4
LT
4222 * call schedule, and on return reacquire the lock.
4223 *
41a2d6cf 4224 * This works OK both with and without CONFIG_PREEMPT. We do strange low-level
1da177e4
LT
4225 * operations here to prevent schedule() from being called twice (once via
4226 * spin_unlock(), once by hand).
4227 */
613afbf8 4228int __cond_resched_lock(spinlock_t *lock)
1da177e4 4229{
d86ee480 4230 int resched = should_resched();
6df3cecb
JK
4231 int ret = 0;
4232
f607c668
PZ
4233 lockdep_assert_held(lock);
4234
4a81e832 4235 if (spin_needbreak(lock) || resched) {
1da177e4 4236 spin_unlock(lock);
d86ee480 4237 if (resched)
95c354fe
NP
4238 __cond_resched();
4239 else
4240 cpu_relax();
6df3cecb 4241 ret = 1;
1da177e4 4242 spin_lock(lock);
1da177e4 4243 }
6df3cecb 4244 return ret;
1da177e4 4245}
613afbf8 4246EXPORT_SYMBOL(__cond_resched_lock);
1da177e4 4247
613afbf8 4248int __sched __cond_resched_softirq(void)
1da177e4
LT
4249{
4250 BUG_ON(!in_softirq());
4251
d86ee480 4252 if (should_resched()) {
98d82567 4253 local_bh_enable();
1da177e4
LT
4254 __cond_resched();
4255 local_bh_disable();
4256 return 1;
4257 }
4258 return 0;
4259}
613afbf8 4260EXPORT_SYMBOL(__cond_resched_softirq);
1da177e4 4261
1da177e4
LT
4262/**
4263 * yield - yield the current processor to other threads.
4264 *
8e3fabfd
PZ
4265 * Do not ever use this function, there's a 99% chance you're doing it wrong.
4266 *
4267 * The scheduler is at all times free to pick the calling task as the most
4268 * eligible task to run, if removing the yield() call from your code breaks
4269 * it, its already broken.
4270 *
4271 * Typical broken usage is:
4272 *
4273 * while (!event)
4274 * yield();
4275 *
4276 * where one assumes that yield() will let 'the other' process run that will
4277 * make event true. If the current task is a SCHED_FIFO task that will never
4278 * happen. Never use yield() as a progress guarantee!!
4279 *
4280 * If you want to use yield() to wait for something, use wait_event().
4281 * If you want to use yield() to be 'nice' for others, use cond_resched().
4282 * If you still want to use yield(), do not!
1da177e4
LT
4283 */
4284void __sched yield(void)
4285{
4286 set_current_state(TASK_RUNNING);
4287 sys_sched_yield();
4288}
1da177e4
LT
4289EXPORT_SYMBOL(yield);
4290
d95f4122
MG
4291/**
4292 * yield_to - yield the current processor to another thread in
4293 * your thread group, or accelerate that thread toward the
4294 * processor it's on.
16addf95
RD
4295 * @p: target task
4296 * @preempt: whether task preemption is allowed or not
d95f4122
MG
4297 *
4298 * It's the caller's job to ensure that the target task struct
4299 * can't go away on us before we can do any checks.
4300 *
e69f6186 4301 * Return:
7b270f60
PZ
4302 * true (>0) if we indeed boosted the target task.
4303 * false (0) if we failed to boost the target.
4304 * -ESRCH if there's no task to yield to.
d95f4122 4305 */
fa93384f 4306int __sched yield_to(struct task_struct *p, bool preempt)
d95f4122
MG
4307{
4308 struct task_struct *curr = current;
4309 struct rq *rq, *p_rq;
4310 unsigned long flags;
c3c18640 4311 int yielded = 0;
d95f4122
MG
4312
4313 local_irq_save(flags);
4314 rq = this_rq();
4315
4316again:
4317 p_rq = task_rq(p);
7b270f60
PZ
4318 /*
4319 * If we're the only runnable task on the rq and target rq also
4320 * has only one task, there's absolutely no point in yielding.
4321 */
4322 if (rq->nr_running == 1 && p_rq->nr_running == 1) {
4323 yielded = -ESRCH;
4324 goto out_irq;
4325 }
4326
d95f4122 4327 double_rq_lock(rq, p_rq);
39e24d8f 4328 if (task_rq(p) != p_rq) {
d95f4122
MG
4329 double_rq_unlock(rq, p_rq);
4330 goto again;
4331 }
4332
4333 if (!curr->sched_class->yield_to_task)
7b270f60 4334 goto out_unlock;
d95f4122
MG
4335
4336 if (curr->sched_class != p->sched_class)
7b270f60 4337 goto out_unlock;
d95f4122
MG
4338
4339 if (task_running(p_rq, p) || p->state)
7b270f60 4340 goto out_unlock;
d95f4122
MG
4341
4342 yielded = curr->sched_class->yield_to_task(rq, p, preempt);
6d1cafd8 4343 if (yielded) {
d95f4122 4344 schedstat_inc(rq, yld_count);
6d1cafd8
VP
4345 /*
4346 * Make p's CPU reschedule; pick_next_entity takes care of
4347 * fairness.
4348 */
4349 if (preempt && rq != p_rq)
8875125e 4350 resched_curr(p_rq);
6d1cafd8 4351 }
d95f4122 4352
7b270f60 4353out_unlock:
d95f4122 4354 double_rq_unlock(rq, p_rq);
7b270f60 4355out_irq:
d95f4122
MG
4356 local_irq_restore(flags);
4357
7b270f60 4358 if (yielded > 0)
d95f4122
MG
4359 schedule();
4360
4361 return yielded;
4362}
4363EXPORT_SYMBOL_GPL(yield_to);
4364
1da177e4 4365/*
41a2d6cf 4366 * This task is about to go to sleep on IO. Increment rq->nr_iowait so
1da177e4 4367 * that process accounting knows that this is a task in IO wait state.
1da177e4
LT
4368 */
4369void __sched io_schedule(void)
4370{
54d35f29 4371 struct rq *rq = raw_rq();
1da177e4 4372
0ff92245 4373 delayacct_blkio_start();
1da177e4 4374 atomic_inc(&rq->nr_iowait);
73c10101 4375 blk_flush_plug(current);
8f0dfc34 4376 current->in_iowait = 1;
1da177e4 4377 schedule();
8f0dfc34 4378 current->in_iowait = 0;
1da177e4 4379 atomic_dec(&rq->nr_iowait);
0ff92245 4380 delayacct_blkio_end();
1da177e4 4381}
1da177e4
LT
4382EXPORT_SYMBOL(io_schedule);
4383
4384long __sched io_schedule_timeout(long timeout)
4385{
54d35f29 4386 struct rq *rq = raw_rq();
1da177e4
LT
4387 long ret;
4388
0ff92245 4389 delayacct_blkio_start();
1da177e4 4390 atomic_inc(&rq->nr_iowait);
73c10101 4391 blk_flush_plug(current);
8f0dfc34 4392 current->in_iowait = 1;
1da177e4 4393 ret = schedule_timeout(timeout);
8f0dfc34 4394 current->in_iowait = 0;
1da177e4 4395 atomic_dec(&rq->nr_iowait);
0ff92245 4396 delayacct_blkio_end();
1da177e4
LT
4397 return ret;
4398}
4399
4400/**
4401 * sys_sched_get_priority_max - return maximum RT priority.
4402 * @policy: scheduling class.
4403 *
e69f6186
YB
4404 * Return: On success, this syscall returns the maximum
4405 * rt_priority that can be used by a given scheduling class.
4406 * On failure, a negative error code is returned.
1da177e4 4407 */
5add95d4 4408SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
1da177e4
LT
4409{
4410 int ret = -EINVAL;
4411
4412 switch (policy) {
4413 case SCHED_FIFO:
4414 case SCHED_RR:
4415 ret = MAX_USER_RT_PRIO-1;
4416 break;
aab03e05 4417 case SCHED_DEADLINE:
1da177e4 4418 case SCHED_NORMAL:
b0a9499c 4419 case SCHED_BATCH:
dd41f596 4420 case SCHED_IDLE:
1da177e4
LT
4421 ret = 0;
4422 break;
4423 }
4424 return ret;
4425}
4426
4427/**
4428 * sys_sched_get_priority_min - return minimum RT priority.
4429 * @policy: scheduling class.
4430 *
e69f6186
YB
4431 * Return: On success, this syscall returns the minimum
4432 * rt_priority that can be used by a given scheduling class.
4433 * On failure, a negative error code is returned.
1da177e4 4434 */
5add95d4 4435SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
1da177e4
LT
4436{
4437 int ret = -EINVAL;
4438
4439 switch (policy) {
4440 case SCHED_FIFO:
4441 case SCHED_RR:
4442 ret = 1;
4443 break;
aab03e05 4444 case SCHED_DEADLINE:
1da177e4 4445 case SCHED_NORMAL:
b0a9499c 4446 case SCHED_BATCH:
dd41f596 4447 case SCHED_IDLE:
1da177e4
LT
4448 ret = 0;
4449 }
4450 return ret;
4451}
4452
4453/**
4454 * sys_sched_rr_get_interval - return the default timeslice of a process.
4455 * @pid: pid of the process.
4456 * @interval: userspace pointer to the timeslice value.
4457 *
4458 * this syscall writes the default timeslice value of a given process
4459 * into the user-space timespec buffer. A value of '0' means infinity.
e69f6186
YB
4460 *
4461 * Return: On success, 0 and the timeslice is in @interval. Otherwise,
4462 * an error code.
1da177e4 4463 */
17da2bd9 4464SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
754fe8d2 4465 struct timespec __user *, interval)
1da177e4 4466{
36c8b586 4467 struct task_struct *p;
a4ec24b4 4468 unsigned int time_slice;
dba091b9
TG
4469 unsigned long flags;
4470 struct rq *rq;
3a5c359a 4471 int retval;
1da177e4 4472 struct timespec t;
1da177e4
LT
4473
4474 if (pid < 0)
3a5c359a 4475 return -EINVAL;
1da177e4
LT
4476
4477 retval = -ESRCH;
1a551ae7 4478 rcu_read_lock();
1da177e4
LT
4479 p = find_process_by_pid(pid);
4480 if (!p)
4481 goto out_unlock;
4482
4483 retval = security_task_getscheduler(p);
4484 if (retval)
4485 goto out_unlock;
4486
dba091b9 4487 rq = task_rq_lock(p, &flags);
a57beec5
PZ
4488 time_slice = 0;
4489 if (p->sched_class->get_rr_interval)
4490 time_slice = p->sched_class->get_rr_interval(rq, p);
0122ec5b 4491 task_rq_unlock(rq, p, &flags);
a4ec24b4 4492
1a551ae7 4493 rcu_read_unlock();
a4ec24b4 4494 jiffies_to_timespec(time_slice, &t);
1da177e4 4495 retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
1da177e4 4496 return retval;
3a5c359a 4497
1da177e4 4498out_unlock:
1a551ae7 4499 rcu_read_unlock();
1da177e4
LT
4500 return retval;
4501}
4502
7c731e0a 4503static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
36c8b586 4504
82a1fcb9 4505void sched_show_task(struct task_struct *p)
1da177e4 4506{
1da177e4 4507 unsigned long free = 0;
4e79752c 4508 int ppid;
36c8b586 4509 unsigned state;
1da177e4 4510
1da177e4 4511 state = p->state ? __ffs(p->state) + 1 : 0;
28d0686c 4512 printk(KERN_INFO "%-15.15s %c", p->comm,
2ed6e34f 4513 state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
4bd77321 4514#if BITS_PER_LONG == 32
1da177e4 4515 if (state == TASK_RUNNING)
3df0fc5b 4516 printk(KERN_CONT " running ");
1da177e4 4517 else
3df0fc5b 4518 printk(KERN_CONT " %08lx ", thread_saved_pc(p));
1da177e4
LT
4519#else
4520 if (state == TASK_RUNNING)
3df0fc5b 4521 printk(KERN_CONT " running task ");
1da177e4 4522 else
3df0fc5b 4523 printk(KERN_CONT " %016lx ", thread_saved_pc(p));
1da177e4
LT
4524#endif
4525#ifdef CONFIG_DEBUG_STACK_USAGE
7c9f8861 4526 free = stack_not_used(p);
1da177e4 4527#endif
4e79752c
PM
4528 rcu_read_lock();
4529 ppid = task_pid_nr(rcu_dereference(p->real_parent));
4530 rcu_read_unlock();
3df0fc5b 4531 printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
4e79752c 4532 task_pid_nr(p), ppid,
aa47b7e0 4533 (unsigned long)task_thread_info(p)->flags);
1da177e4 4534
3d1cb205 4535 print_worker_info(KERN_INFO, p);
5fb5e6de 4536 show_stack(p, NULL);
1da177e4
LT
4537}
4538
e59e2ae2 4539void show_state_filter(unsigned long state_filter)
1da177e4 4540{
36c8b586 4541 struct task_struct *g, *p;
1da177e4 4542
4bd77321 4543#if BITS_PER_LONG == 32
3df0fc5b
PZ
4544 printk(KERN_INFO
4545 " task PC stack pid father\n");
1da177e4 4546#else
3df0fc5b
PZ
4547 printk(KERN_INFO
4548 " task PC stack pid father\n");
1da177e4 4549#endif
510f5acc 4550 rcu_read_lock();
5d07f420 4551 for_each_process_thread(g, p) {
1da177e4
LT
4552 /*
4553 * reset the NMI-timeout, listing all files on a slow
25985edc 4554 * console might take a lot of time:
1da177e4
LT
4555 */
4556 touch_nmi_watchdog();
39bc89fd 4557 if (!state_filter || (p->state & state_filter))
82a1fcb9 4558 sched_show_task(p);
5d07f420 4559 }
1da177e4 4560
04c9167f
JF
4561 touch_all_softlockup_watchdogs();
4562
dd41f596
IM
4563#ifdef CONFIG_SCHED_DEBUG
4564 sysrq_sched_debug_show();
4565#endif
510f5acc 4566 rcu_read_unlock();
e59e2ae2
IM
4567 /*
4568 * Only show locks if all tasks are dumped:
4569 */
93335a21 4570 if (!state_filter)
e59e2ae2 4571 debug_show_all_locks();
1da177e4
LT
4572}
4573
0db0628d 4574void init_idle_bootup_task(struct task_struct *idle)
1df21055 4575{
dd41f596 4576 idle->sched_class = &idle_sched_class;
1df21055
IM
4577}
4578
f340c0d1
IM
4579/**
4580 * init_idle - set up an idle thread for a given CPU
4581 * @idle: task in question
4582 * @cpu: cpu the idle task belongs to
4583 *
4584 * NOTE: this function does not set the idle thread's NEED_RESCHED
4585 * flag, to make booting more robust.
4586 */
0db0628d 4587void init_idle(struct task_struct *idle, int cpu)
1da177e4 4588{
70b97a7f 4589 struct rq *rq = cpu_rq(cpu);
1da177e4
LT
4590 unsigned long flags;
4591
05fa785c 4592 raw_spin_lock_irqsave(&rq->lock, flags);
5cbd54ef 4593
5e1576ed 4594 __sched_fork(0, idle);
06b83b5f 4595 idle->state = TASK_RUNNING;
dd41f596
IM
4596 idle->se.exec_start = sched_clock();
4597
1e1b6c51 4598 do_set_cpus_allowed(idle, cpumask_of(cpu));
6506cf6c
PZ
4599 /*
4600 * We're having a chicken and egg problem, even though we are
4601 * holding rq->lock, the cpu isn't yet set to this cpu so the
4602 * lockdep check in task_group() will fail.
4603 *
4604 * Similar case to sched_fork(). / Alternatively we could
4605 * use task_rq_lock() here and obtain the other rq->lock.
4606 *
4607 * Silence PROVE_RCU
4608 */
4609 rcu_read_lock();
dd41f596 4610 __set_task_cpu(idle, cpu);
6506cf6c 4611 rcu_read_unlock();
1da177e4 4612
1da177e4 4613 rq->curr = rq->idle = idle;
da0c1e65 4614 idle->on_rq = TASK_ON_RQ_QUEUED;
3ca7a440
PZ
4615#if defined(CONFIG_SMP)
4616 idle->on_cpu = 1;
4866cde0 4617#endif
05fa785c 4618 raw_spin_unlock_irqrestore(&rq->lock, flags);
1da177e4
LT
4619
4620 /* Set the preempt count _outside_ the spinlocks! */
01028747 4621 init_idle_preempt_count(idle, cpu);
55cd5340 4622
dd41f596
IM
4623 /*
4624 * The idle tasks have their own, simple scheduling class:
4625 */
4626 idle->sched_class = &idle_sched_class;
868baf07 4627 ftrace_graph_init_idle_task(idle, cpu);
45eacc69 4628 vtime_init_idle(idle, cpu);
f1c6f1a7
CE
4629#if defined(CONFIG_SMP)
4630 sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu);
4631#endif
19978ca6
IM
4632}
4633
1da177e4 4634#ifdef CONFIG_SMP
a15b12ac
KT
4635/*
4636 * move_queued_task - move a queued task to new rq.
4637 *
4638 * Returns (locked) new rq. Old rq's lock is released.
4639 */
4640static struct rq *move_queued_task(struct task_struct *p, int new_cpu)
4641{
4642 struct rq *rq = task_rq(p);
4643
4644 lockdep_assert_held(&rq->lock);
4645
4646 dequeue_task(rq, p, 0);
4647 p->on_rq = TASK_ON_RQ_MIGRATING;
4648 set_task_cpu(p, new_cpu);
4649 raw_spin_unlock(&rq->lock);
4650
4651 rq = cpu_rq(new_cpu);
4652
4653 raw_spin_lock(&rq->lock);
4654 BUG_ON(task_cpu(p) != new_cpu);
4655 p->on_rq = TASK_ON_RQ_QUEUED;
4656 enqueue_task(rq, p, 0);
4657 check_preempt_curr(rq, p, 0);
4658
4659 return rq;
4660}
4661
1e1b6c51
KM
4662void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
4663{
4664 if (p->sched_class && p->sched_class->set_cpus_allowed)
4665 p->sched_class->set_cpus_allowed(p, new_mask);
4939602a
PZ
4666
4667 cpumask_copy(&p->cpus_allowed, new_mask);
29baa747 4668 p->nr_cpus_allowed = cpumask_weight(new_mask);
1e1b6c51
KM
4669}
4670
1da177e4
LT
4671/*
4672 * This is how migration works:
4673 *
969c7921
TH
4674 * 1) we invoke migration_cpu_stop() on the target CPU using
4675 * stop_one_cpu().
4676 * 2) stopper starts to run (implicitly forcing the migrated thread
4677 * off the CPU)
4678 * 3) it checks whether the migrated task is still in the wrong runqueue.
4679 * 4) if it's in the wrong runqueue then the migration thread removes
1da177e4 4680 * it and puts it into the right queue.
969c7921
TH
4681 * 5) stopper completes and stop_one_cpu() returns and the migration
4682 * is done.
1da177e4
LT
4683 */
4684
4685/*
4686 * Change a given task's CPU affinity. Migrate the thread to a
4687 * proper CPU and schedule it away if the CPU it's executing on
4688 * is removed from the allowed bitmask.
4689 *
4690 * NOTE: the caller must have a valid reference to the task, the
41a2d6cf 4691 * task must not exit() & deallocate itself prematurely. The
1da177e4
LT
4692 * call is not atomic; no spinlocks may be held.
4693 */
96f874e2 4694int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
1da177e4
LT
4695{
4696 unsigned long flags;
70b97a7f 4697 struct rq *rq;
969c7921 4698 unsigned int dest_cpu;
48f24c4d 4699 int ret = 0;
1da177e4
LT
4700
4701 rq = task_rq_lock(p, &flags);
e2912009 4702
db44fc01
YZ
4703 if (cpumask_equal(&p->cpus_allowed, new_mask))
4704 goto out;
4705
6ad4c188 4706 if (!cpumask_intersects(new_mask, cpu_active_mask)) {
1da177e4
LT
4707 ret = -EINVAL;
4708 goto out;
4709 }
4710
1e1b6c51 4711 do_set_cpus_allowed(p, new_mask);
73fe6aae 4712
1da177e4 4713 /* Can the task run on the task's current CPU? If so, we're done */
96f874e2 4714 if (cpumask_test_cpu(task_cpu(p), new_mask))
1da177e4
LT
4715 goto out;
4716
969c7921 4717 dest_cpu = cpumask_any_and(cpu_active_mask, new_mask);
a15b12ac 4718 if (task_running(rq, p) || p->state == TASK_WAKING) {
969c7921 4719 struct migration_arg arg = { p, dest_cpu };
1da177e4 4720 /* Need help from migration thread: drop lock and wait. */
0122ec5b 4721 task_rq_unlock(rq, p, &flags);
969c7921 4722 stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
1da177e4
LT
4723 tlb_migrate_finish(p->mm);
4724 return 0;
a15b12ac
KT
4725 } else if (task_on_rq_queued(p))
4726 rq = move_queued_task(p, dest_cpu);
1da177e4 4727out:
0122ec5b 4728 task_rq_unlock(rq, p, &flags);
48f24c4d 4729
1da177e4
LT
4730 return ret;
4731}
cd8ba7cd 4732EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
1da177e4
LT
4733
4734/*
41a2d6cf 4735 * Move (not current) task off this cpu, onto dest cpu. We're doing
1da177e4
LT
4736 * this because either it can't run here any more (set_cpus_allowed()
4737 * away from this CPU, or CPU going down), or because we're
4738 * attempting to rebalance this task on exec (sched_exec).
4739 *
4740 * So we race with normal scheduler movements, but that's OK, as long
4741 * as the task is no longer on this CPU.
efc30814
KK
4742 *
4743 * Returns non-zero if task was successfully migrated.
1da177e4 4744 */
efc30814 4745static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
1da177e4 4746{
a1e01829 4747 struct rq *rq;
e2912009 4748 int ret = 0;
1da177e4 4749
e761b772 4750 if (unlikely(!cpu_active(dest_cpu)))
efc30814 4751 return ret;
1da177e4 4752
a1e01829 4753 rq = cpu_rq(src_cpu);
1da177e4 4754
0122ec5b 4755 raw_spin_lock(&p->pi_lock);
a1e01829 4756 raw_spin_lock(&rq->lock);
1da177e4
LT
4757 /* Already moved. */
4758 if (task_cpu(p) != src_cpu)
b1e38734 4759 goto done;
a1e01829 4760
1da177e4 4761 /* Affinity changed (again). */
fa17b507 4762 if (!cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p)))
b1e38734 4763 goto fail;
1da177e4 4764
e2912009
PZ
4765 /*
4766 * If we're not on a rq, the next wake-up will ensure we're
4767 * placed properly.
4768 */
a15b12ac
KT
4769 if (task_on_rq_queued(p))
4770 rq = move_queued_task(p, dest_cpu);
b1e38734 4771done:
efc30814 4772 ret = 1;
b1e38734 4773fail:
a1e01829 4774 raw_spin_unlock(&rq->lock);
0122ec5b 4775 raw_spin_unlock(&p->pi_lock);
efc30814 4776 return ret;
1da177e4
LT
4777}
4778
e6628d5b
MG
4779#ifdef CONFIG_NUMA_BALANCING
4780/* Migrate current task p to target_cpu */
4781int migrate_task_to(struct task_struct *p, int target_cpu)
4782{
4783 struct migration_arg arg = { p, target_cpu };
4784 int curr_cpu = task_cpu(p);
4785
4786 if (curr_cpu == target_cpu)
4787 return 0;
4788
4789 if (!cpumask_test_cpu(target_cpu, tsk_cpus_allowed(p)))
4790 return -EINVAL;
4791
4792 /* TODO: This is not properly updating schedstats */
4793
286549dc 4794 trace_sched_move_numa(p, curr_cpu, target_cpu);
e6628d5b
MG
4795 return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg);
4796}
0ec8aa00
PZ
4797
4798/*
4799 * Requeue a task on a given node and accurately track the number of NUMA
4800 * tasks on the runqueues
4801 */
4802void sched_setnuma(struct task_struct *p, int nid)
4803{
4804 struct rq *rq;
4805 unsigned long flags;
da0c1e65 4806 bool queued, running;
0ec8aa00
PZ
4807
4808 rq = task_rq_lock(p, &flags);
da0c1e65 4809 queued = task_on_rq_queued(p);
0ec8aa00
PZ
4810 running = task_current(rq, p);
4811
da0c1e65 4812 if (queued)
0ec8aa00
PZ
4813 dequeue_task(rq, p, 0);
4814 if (running)
f3cd1c4e 4815 put_prev_task(rq, p);
0ec8aa00
PZ
4816
4817 p->numa_preferred_nid = nid;
0ec8aa00
PZ
4818
4819 if (running)
4820 p->sched_class->set_curr_task(rq);
da0c1e65 4821 if (queued)
0ec8aa00
PZ
4822 enqueue_task(rq, p, 0);
4823 task_rq_unlock(rq, p, &flags);
4824}
e6628d5b
MG
4825#endif
4826
1da177e4 4827/*
969c7921
TH
4828 * migration_cpu_stop - this will be executed by a highprio stopper thread
4829 * and performs thread migration by bumping thread off CPU then
4830 * 'pushing' onto another runqueue.
1da177e4 4831 */
969c7921 4832static int migration_cpu_stop(void *data)
1da177e4 4833{
969c7921 4834 struct migration_arg *arg = data;
f7b4cddc 4835
969c7921
TH
4836 /*
4837 * The original target cpu might have gone down and we might
4838 * be on another cpu but it doesn't matter.
4839 */
f7b4cddc 4840 local_irq_disable();
5cd038f5
LJ
4841 /*
4842 * We need to explicitly wake pending tasks before running
4843 * __migrate_task() such that we will not miss enforcing cpus_allowed
4844 * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test.
4845 */
4846 sched_ttwu_pending();
969c7921 4847 __migrate_task(arg->task, raw_smp_processor_id(), arg->dest_cpu);
f7b4cddc 4848 local_irq_enable();
1da177e4 4849 return 0;
f7b4cddc
ON
4850}
4851
1da177e4 4852#ifdef CONFIG_HOTPLUG_CPU
48c5ccae 4853
054b9108 4854/*
48c5ccae
PZ
4855 * Ensures that the idle task is using init_mm right before its cpu goes
4856 * offline.
054b9108 4857 */
48c5ccae 4858void idle_task_exit(void)
1da177e4 4859{
48c5ccae 4860 struct mm_struct *mm = current->active_mm;
e76bd8d9 4861
48c5ccae 4862 BUG_ON(cpu_online(smp_processor_id()));
e76bd8d9 4863
a53efe5f 4864 if (mm != &init_mm) {
48c5ccae 4865 switch_mm(mm, &init_mm, current);
a53efe5f
MS
4866 finish_arch_post_lock_switch();
4867 }
48c5ccae 4868 mmdrop(mm);
1da177e4
LT
4869}
4870
4871/*
5d180232
PZ
4872 * Since this CPU is going 'away' for a while, fold any nr_active delta
4873 * we might have. Assumes we're called after migrate_tasks() so that the
4874 * nr_active count is stable.
4875 *
4876 * Also see the comment "Global load-average calculations".
1da177e4 4877 */
5d180232 4878static void calc_load_migrate(struct rq *rq)
1da177e4 4879{
5d180232
PZ
4880 long delta = calc_load_fold_active(rq);
4881 if (delta)
4882 atomic_long_add(delta, &calc_load_tasks);
1da177e4
LT
4883}
4884
3f1d2a31
PZ
4885static void put_prev_task_fake(struct rq *rq, struct task_struct *prev)
4886{
4887}
4888
4889static const struct sched_class fake_sched_class = {
4890 .put_prev_task = put_prev_task_fake,
4891};
4892
4893static struct task_struct fake_task = {
4894 /*
4895 * Avoid pull_{rt,dl}_task()
4896 */
4897 .prio = MAX_PRIO + 1,
4898 .sched_class = &fake_sched_class,
4899};
4900
48f24c4d 4901/*
48c5ccae
PZ
4902 * Migrate all tasks from the rq, sleeping tasks will be migrated by
4903 * try_to_wake_up()->select_task_rq().
4904 *
4905 * Called with rq->lock held even though we'er in stop_machine() and
4906 * there's no concurrency possible, we hold the required locks anyway
4907 * because of lock validation efforts.
1da177e4 4908 */
48c5ccae 4909static void migrate_tasks(unsigned int dead_cpu)
1da177e4 4910{
70b97a7f 4911 struct rq *rq = cpu_rq(dead_cpu);
48c5ccae
PZ
4912 struct task_struct *next, *stop = rq->stop;
4913 int dest_cpu;
1da177e4
LT
4914
4915 /*
48c5ccae
PZ
4916 * Fudge the rq selection such that the below task selection loop
4917 * doesn't get stuck on the currently eligible stop task.
4918 *
4919 * We're currently inside stop_machine() and the rq is either stuck
4920 * in the stop_machine_cpu_stop() loop, or we're executing this code,
4921 * either way we should never end up calling schedule() until we're
4922 * done here.
1da177e4 4923 */
48c5ccae 4924 rq->stop = NULL;
48f24c4d 4925
77bd3970
FW
4926 /*
4927 * put_prev_task() and pick_next_task() sched
4928 * class method both need to have an up-to-date
4929 * value of rq->clock[_task]
4930 */
4931 update_rq_clock(rq);
4932
dd41f596 4933 for ( ; ; ) {
48c5ccae
PZ
4934 /*
4935 * There's this thread running, bail when that's the only
4936 * remaining thread.
4937 */
4938 if (rq->nr_running == 1)
dd41f596 4939 break;
48c5ccae 4940
3f1d2a31 4941 next = pick_next_task(rq, &fake_task);
48c5ccae 4942 BUG_ON(!next);
79c53799 4943 next->sched_class->put_prev_task(rq, next);
e692ab53 4944
48c5ccae
PZ
4945 /* Find suitable destination for @next, with force if needed. */
4946 dest_cpu = select_fallback_rq(dead_cpu, next);
4947 raw_spin_unlock(&rq->lock);
4948
4949 __migrate_task(next, dead_cpu, dest_cpu);
4950
4951 raw_spin_lock(&rq->lock);
1da177e4 4952 }
dce48a84 4953
48c5ccae 4954 rq->stop = stop;
dce48a84 4955}
48c5ccae 4956
1da177e4
LT
4957#endif /* CONFIG_HOTPLUG_CPU */
4958
e692ab53
NP
4959#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)
4960
4961static struct ctl_table sd_ctl_dir[] = {
e0361851
AD
4962 {
4963 .procname = "sched_domain",
c57baf1e 4964 .mode = 0555,
e0361851 4965 },
56992309 4966 {}
e692ab53
NP
4967};
4968
4969static struct ctl_table sd_ctl_root[] = {
e0361851
AD
4970 {
4971 .procname = "kernel",
c57baf1e 4972 .mode = 0555,
e0361851
AD
4973 .child = sd_ctl_dir,
4974 },
56992309 4975 {}
e692ab53
NP
4976};
4977
4978static struct ctl_table *sd_alloc_ctl_entry(int n)
4979{
4980 struct ctl_table *entry =
5cf9f062 4981 kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
e692ab53 4982
e692ab53
NP
4983 return entry;
4984}
4985
6382bc90
MM
4986static void sd_free_ctl_entry(struct ctl_table **tablep)
4987{
cd790076 4988 struct ctl_table *entry;
6382bc90 4989
cd790076
MM
4990 /*
4991 * In the intermediate directories, both the child directory and
4992 * procname are dynamically allocated and could fail but the mode
41a2d6cf 4993 * will always be set. In the lowest directory the names are
cd790076
MM
4994 * static strings and all have proc handlers.
4995 */
4996 for (entry = *tablep; entry->mode; entry++) {
6382bc90
MM
4997 if (entry->child)
4998 sd_free_ctl_entry(&entry->child);
cd790076
MM
4999 if (entry->proc_handler == NULL)
5000 kfree(entry->procname);
5001 }
6382bc90
MM
5002
5003 kfree(*tablep);
5004 *tablep = NULL;
5005}
5006
201c373e 5007static int min_load_idx = 0;
fd9b86d3 5008static int max_load_idx = CPU_LOAD_IDX_MAX-1;
201c373e 5009
e692ab53 5010static void
e0361851 5011set_table_entry(struct ctl_table *entry,
e692ab53 5012 const char *procname, void *data, int maxlen,
201c373e
NK
5013 umode_t mode, proc_handler *proc_handler,
5014 bool load_idx)
e692ab53 5015{
e692ab53
NP
5016 entry->procname = procname;
5017 entry->data = data;
5018 entry->maxlen = maxlen;
5019 entry->mode = mode;
5020 entry->proc_handler = proc_handler;
201c373e
NK
5021
5022 if (load_idx) {
5023 entry->extra1 = &min_load_idx;
5024 entry->extra2 = &max_load_idx;
5025 }
e692ab53
NP
5026}
5027
5028static struct ctl_table *
5029sd_alloc_ctl_domain_table(struct sched_domain *sd)
5030{
37e6bae8 5031 struct ctl_table *table = sd_alloc_ctl_entry(14);
e692ab53 5032
ad1cdc1d
MM
5033 if (table == NULL)
5034 return NULL;
5035
e0361851 5036 set_table_entry(&table[0], "min_interval", &sd->min_interval,
201c373e 5037 sizeof(long), 0644, proc_doulongvec_minmax, false);
e0361851 5038 set_table_entry(&table[1], "max_interval", &sd->max_interval,
201c373e 5039 sizeof(long), 0644, proc_doulongvec_minmax, false);
e0361851 5040 set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
201c373e 5041 sizeof(int), 0644, proc_dointvec_minmax, true);
e0361851 5042 set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
201c373e 5043 sizeof(int), 0644, proc_dointvec_minmax, true);
e0361851 5044 set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
201c373e 5045 sizeof(int), 0644, proc_dointvec_minmax, true);
e0361851 5046 set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
201c373e 5047 sizeof(int), 0644, proc_dointvec_minmax, true);
e0361851 5048 set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
201c373e 5049 sizeof(int), 0644, proc_dointvec_minmax, true);
e0361851 5050 set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
201c373e 5051 sizeof(int), 0644, proc_dointvec_minmax, false);
e0361851 5052 set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
201c373e 5053 sizeof(int), 0644, proc_dointvec_minmax, false);
ace8b3d6 5054 set_table_entry(&table[9], "cache_nice_tries",
e692ab53 5055 &sd->cache_nice_tries,
201c373e 5056 sizeof(int), 0644, proc_dointvec_minmax, false);
ace8b3d6 5057 set_table_entry(&table[10], "flags", &sd->flags,
201c373e 5058 sizeof(int), 0644, proc_dointvec_minmax, false);
37e6bae8
AS
5059 set_table_entry(&table[11], "max_newidle_lb_cost",
5060 &sd->max_newidle_lb_cost,
5061 sizeof(long), 0644, proc_doulongvec_minmax, false);
5062 set_table_entry(&table[12], "name", sd->name,
201c373e 5063 CORENAME_MAX_SIZE, 0444, proc_dostring, false);
37e6bae8 5064 /* &table[13] is terminator */
e692ab53
NP
5065
5066 return table;
5067}
5068
be7002e6 5069static struct ctl_table *sd_alloc_ctl_cpu_table(int cpu)
e692ab53
NP
5070{
5071 struct ctl_table *entry, *table;
5072 struct sched_domain *sd;
5073 int domain_num = 0, i;
5074 char buf[32];
5075
5076 for_each_domain(cpu, sd)
5077 domain_num++;
5078 entry = table = sd_alloc_ctl_entry(domain_num + 1);
ad1cdc1d
MM
5079 if (table == NULL)
5080 return NULL;
e692ab53
NP
5081
5082 i = 0;
5083 for_each_domain(cpu, sd) {
5084 snprintf(buf, 32, "domain%d", i);
e692ab53 5085 entry->procname = kstrdup(buf, GFP_KERNEL);
c57baf1e 5086 entry->mode = 0555;
e692ab53
NP
5087 entry->child = sd_alloc_ctl_domain_table(sd);
5088 entry++;
5089 i++;
5090 }
5091 return table;
5092}
5093
5094static struct ctl_table_header *sd_sysctl_header;
6382bc90 5095static void register_sched_domain_sysctl(void)
e692ab53 5096{
6ad4c188 5097 int i, cpu_num = num_possible_cpus();
e692ab53
NP
5098 struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
5099 char buf[32];
5100
7378547f
MM
5101 WARN_ON(sd_ctl_dir[0].child);
5102 sd_ctl_dir[0].child = entry;
5103
ad1cdc1d
MM
5104 if (entry == NULL)
5105 return;
5106
6ad4c188 5107 for_each_possible_cpu(i) {
e692ab53 5108 snprintf(buf, 32, "cpu%d", i);
e692ab53 5109 entry->procname = kstrdup(buf, GFP_KERNEL);
c57baf1e 5110 entry->mode = 0555;
e692ab53 5111 entry->child = sd_alloc_ctl_cpu_table(i);
97b6ea7b 5112 entry++;
e692ab53 5113 }
7378547f
MM
5114
5115 WARN_ON(sd_sysctl_header);
e692ab53
NP
5116 sd_sysctl_header = register_sysctl_table(sd_ctl_root);
5117}
6382bc90 5118
7378547f 5119/* may be called multiple times per register */
6382bc90
MM
5120static void unregister_sched_domain_sysctl(void)
5121{
7378547f
MM
5122 if (sd_sysctl_header)
5123 unregister_sysctl_table(sd_sysctl_header);
6382bc90 5124 sd_sysctl_header = NULL;
7378547f
MM
5125 if (sd_ctl_dir[0].child)
5126 sd_free_ctl_entry(&sd_ctl_dir[0].child);
6382bc90 5127}
e692ab53 5128#else
6382bc90
MM
5129static void register_sched_domain_sysctl(void)
5130{
5131}
5132static void unregister_sched_domain_sysctl(void)
e692ab53
NP
5133{
5134}
5135#endif
5136
1f11eb6a
GH
5137static void set_rq_online(struct rq *rq)
5138{
5139 if (!rq->online) {
5140 const struct sched_class *class;
5141
c6c4927b 5142 cpumask_set_cpu(rq->cpu, rq->rd->online);
1f11eb6a
GH
5143 rq->online = 1;
5144
5145 for_each_class(class) {
5146 if (class->rq_online)
5147 class->rq_online(rq);
5148 }
5149 }
5150}
5151
5152static void set_rq_offline(struct rq *rq)
5153{
5154 if (rq->online) {
5155 const struct sched_class *class;
5156
5157 for_each_class(class) {
5158 if (class->rq_offline)
5159 class->rq_offline(rq);
5160 }
5161
c6c4927b 5162 cpumask_clear_cpu(rq->cpu, rq->rd->online);
1f11eb6a
GH
5163 rq->online = 0;
5164 }
5165}
5166
1da177e4
LT
5167/*
5168 * migration_call - callback that gets triggered when a CPU is added.
5169 * Here we can start up the necessary migration thread for the new CPU.
5170 */
0db0628d 5171static int
48f24c4d 5172migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
1da177e4 5173{
48f24c4d 5174 int cpu = (long)hcpu;
1da177e4 5175 unsigned long flags;
969c7921 5176 struct rq *rq = cpu_rq(cpu);
1da177e4 5177
48c5ccae 5178 switch (action & ~CPU_TASKS_FROZEN) {
5be9361c 5179
1da177e4 5180 case CPU_UP_PREPARE:
a468d389 5181 rq->calc_load_update = calc_load_update;
1da177e4 5182 break;
48f24c4d 5183
1da177e4 5184 case CPU_ONLINE:
1f94ef59 5185 /* Update our root-domain */
05fa785c 5186 raw_spin_lock_irqsave(&rq->lock, flags);
1f94ef59 5187 if (rq->rd) {
c6c4927b 5188 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
1f11eb6a
GH
5189
5190 set_rq_online(rq);
1f94ef59 5191 }
05fa785c 5192 raw_spin_unlock_irqrestore(&rq->lock, flags);
1da177e4 5193 break;
48f24c4d 5194
1da177e4 5195#ifdef CONFIG_HOTPLUG_CPU
08f503b0 5196 case CPU_DYING:
317f3941 5197 sched_ttwu_pending();
57d885fe 5198 /* Update our root-domain */
05fa785c 5199 raw_spin_lock_irqsave(&rq->lock, flags);
57d885fe 5200 if (rq->rd) {
c6c4927b 5201 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
1f11eb6a 5202 set_rq_offline(rq);
57d885fe 5203 }
48c5ccae
PZ
5204 migrate_tasks(cpu);
5205 BUG_ON(rq->nr_running != 1); /* the migration thread */
05fa785c 5206 raw_spin_unlock_irqrestore(&rq->lock, flags);
5d180232 5207 break;
48c5ccae 5208
5d180232 5209 case CPU_DEAD:
f319da0c 5210 calc_load_migrate(rq);
57d885fe 5211 break;
1da177e4
LT
5212#endif
5213 }
49c022e6
PZ
5214
5215 update_max_interval();
5216
1da177e4
LT
5217 return NOTIFY_OK;
5218}
5219
f38b0820
PM
5220/*
5221 * Register at high priority so that task migration (migrate_all_tasks)
5222 * happens before everything else. This has to be lower priority than
cdd6c482 5223 * the notifier in the perf_event subsystem, though.
1da177e4 5224 */
0db0628d 5225static struct notifier_block migration_notifier = {
1da177e4 5226 .notifier_call = migration_call,
50a323b7 5227 .priority = CPU_PRI_MIGRATION,
1da177e4
LT
5228};
5229
a803f026
CM
5230static void __cpuinit set_cpu_rq_start_time(void)
5231{
5232 int cpu = smp_processor_id();
5233 struct rq *rq = cpu_rq(cpu);
5234 rq->age_stamp = sched_clock_cpu(cpu);
5235}
5236
0db0628d 5237static int sched_cpu_active(struct notifier_block *nfb,
3a101d05
TH
5238 unsigned long action, void *hcpu)
5239{
5240 switch (action & ~CPU_TASKS_FROZEN) {
a803f026
CM
5241 case CPU_STARTING:
5242 set_cpu_rq_start_time();
5243 return NOTIFY_OK;
3a101d05
TH
5244 case CPU_DOWN_FAILED:
5245 set_cpu_active((long)hcpu, true);
5246 return NOTIFY_OK;
5247 default:
5248 return NOTIFY_DONE;
5249 }
5250}
5251
0db0628d 5252static int sched_cpu_inactive(struct notifier_block *nfb,
3a101d05
TH
5253 unsigned long action, void *hcpu)
5254{
de212f18
PZ
5255 unsigned long flags;
5256 long cpu = (long)hcpu;
5257
3a101d05
TH
5258 switch (action & ~CPU_TASKS_FROZEN) {
5259 case CPU_DOWN_PREPARE:
de212f18
PZ
5260 set_cpu_active(cpu, false);
5261
5262 /* explicitly allow suspend */
5263 if (!(action & CPU_TASKS_FROZEN)) {
5264 struct dl_bw *dl_b = dl_bw_of(cpu);
5265 bool overflow;
5266 int cpus;
5267
5268 raw_spin_lock_irqsave(&dl_b->lock, flags);
5269 cpus = dl_bw_cpus(cpu);
5270 overflow = __dl_overflow(dl_b, cpus, 0, 0);
5271 raw_spin_unlock_irqrestore(&dl_b->lock, flags);
5272
5273 if (overflow)
5274 return notifier_from_errno(-EBUSY);
5275 }
3a101d05 5276 return NOTIFY_OK;
3a101d05 5277 }
de212f18
PZ
5278
5279 return NOTIFY_DONE;
3a101d05
TH
5280}
5281
7babe8db 5282static int __init migration_init(void)
1da177e4
LT
5283{
5284 void *cpu = (void *)(long)smp_processor_id();
07dccf33 5285 int err;
48f24c4d 5286
3a101d05 5287 /* Initialize migration for the boot CPU */
07dccf33
AM
5288 err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
5289 BUG_ON(err == NOTIFY_BAD);
1da177e4
LT
5290 migration_call(&migration_notifier, CPU_ONLINE, cpu);
5291 register_cpu_notifier(&migration_notifier);
7babe8db 5292
3a101d05
TH
5293 /* Register cpu active notifiers */
5294 cpu_notifier(sched_cpu_active, CPU_PRI_SCHED_ACTIVE);
5295 cpu_notifier(sched_cpu_inactive, CPU_PRI_SCHED_INACTIVE);
5296
a004cd42 5297 return 0;
1da177e4 5298}
7babe8db 5299early_initcall(migration_init);
1da177e4
LT
5300#endif
5301
5302#ifdef CONFIG_SMP
476f3534 5303
4cb98839
PZ
5304static cpumask_var_t sched_domains_tmpmask; /* sched_domains_mutex */
5305
3e9830dc 5306#ifdef CONFIG_SCHED_DEBUG
4dcf6aff 5307
d039ac60 5308static __read_mostly int sched_debug_enabled;
f6630114 5309
d039ac60 5310static int __init sched_debug_setup(char *str)
f6630114 5311{
d039ac60 5312 sched_debug_enabled = 1;
f6630114
MT
5313
5314 return 0;
5315}
d039ac60
PZ
5316early_param("sched_debug", sched_debug_setup);
5317
5318static inline bool sched_debug(void)
5319{
5320 return sched_debug_enabled;
5321}
f6630114 5322
7c16ec58 5323static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
96f874e2 5324 struct cpumask *groupmask)
1da177e4 5325{
4dcf6aff 5326 struct sched_group *group = sd->groups;
434d53b0 5327 char str[256];
1da177e4 5328
968ea6d8 5329 cpulist_scnprintf(str, sizeof(str), sched_domain_span(sd));
96f874e2 5330 cpumask_clear(groupmask);
4dcf6aff
IM
5331
5332 printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
5333
5334 if (!(sd->flags & SD_LOAD_BALANCE)) {
3df0fc5b 5335 printk("does not load-balance\n");
4dcf6aff 5336 if (sd->parent)
3df0fc5b
PZ
5337 printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
5338 " has parent");
4dcf6aff 5339 return -1;
41c7ce9a
NP
5340 }
5341
3df0fc5b 5342 printk(KERN_CONT "span %s level %s\n", str, sd->name);
4dcf6aff 5343
758b2cdc 5344 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
3df0fc5b
PZ
5345 printk(KERN_ERR "ERROR: domain->span does not contain "
5346 "CPU%d\n", cpu);
4dcf6aff 5347 }
758b2cdc 5348 if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
3df0fc5b
PZ
5349 printk(KERN_ERR "ERROR: domain->groups does not contain"
5350 " CPU%d\n", cpu);
4dcf6aff 5351 }
1da177e4 5352
4dcf6aff 5353 printk(KERN_DEBUG "%*s groups:", level + 1, "");
1da177e4 5354 do {
4dcf6aff 5355 if (!group) {
3df0fc5b
PZ
5356 printk("\n");
5357 printk(KERN_ERR "ERROR: group is NULL\n");
1da177e4
LT
5358 break;
5359 }
5360
c3decf0d 5361 /*
63b2ca30
NP
5362 * Even though we initialize ->capacity to something semi-sane,
5363 * we leave capacity_orig unset. This allows us to detect if
c3decf0d
PZ
5364 * domain iteration is still funny without causing /0 traps.
5365 */
63b2ca30 5366 if (!group->sgc->capacity_orig) {
3df0fc5b 5367 printk(KERN_CONT "\n");
63b2ca30 5368 printk(KERN_ERR "ERROR: domain->cpu_capacity not set\n");
4dcf6aff
IM
5369 break;
5370 }
1da177e4 5371
758b2cdc 5372 if (!cpumask_weight(sched_group_cpus(group))) {
3df0fc5b
PZ
5373 printk(KERN_CONT "\n");
5374 printk(KERN_ERR "ERROR: empty group\n");
4dcf6aff
IM
5375 break;
5376 }
1da177e4 5377
cb83b629
PZ
5378 if (!(sd->flags & SD_OVERLAP) &&
5379 cpumask_intersects(groupmask, sched_group_cpus(group))) {
3df0fc5b
PZ
5380 printk(KERN_CONT "\n");
5381 printk(KERN_ERR "ERROR: repeated CPUs\n");
4dcf6aff
IM
5382 break;
5383 }
1da177e4 5384
758b2cdc 5385 cpumask_or(groupmask, groupmask, sched_group_cpus(group));
1da177e4 5386
968ea6d8 5387 cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
381512cf 5388
3df0fc5b 5389 printk(KERN_CONT " %s", str);
ca8ce3d0 5390 if (group->sgc->capacity != SCHED_CAPACITY_SCALE) {
63b2ca30
NP
5391 printk(KERN_CONT " (cpu_capacity = %d)",
5392 group->sgc->capacity);
381512cf 5393 }
1da177e4 5394
4dcf6aff
IM
5395 group = group->next;
5396 } while (group != sd->groups);
3df0fc5b 5397 printk(KERN_CONT "\n");
1da177e4 5398
758b2cdc 5399 if (!cpumask_equal(sched_domain_span(sd), groupmask))
3df0fc5b 5400 printk(KERN_ERR "ERROR: groups don't span domain->span\n");
1da177e4 5401
758b2cdc
RR
5402 if (sd->parent &&
5403 !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
3df0fc5b
PZ
5404 printk(KERN_ERR "ERROR: parent span is not a superset "
5405 "of domain->span\n");
4dcf6aff
IM
5406 return 0;
5407}
1da177e4 5408
4dcf6aff
IM
5409static void sched_domain_debug(struct sched_domain *sd, int cpu)
5410{
5411 int level = 0;
1da177e4 5412
d039ac60 5413 if (!sched_debug_enabled)
f6630114
MT
5414 return;
5415
4dcf6aff
IM
5416 if (!sd) {
5417 printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
5418 return;
5419 }
1da177e4 5420
4dcf6aff
IM
5421 printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
5422
5423 for (;;) {
4cb98839 5424 if (sched_domain_debug_one(sd, cpu, level, sched_domains_tmpmask))
4dcf6aff 5425 break;
1da177e4
LT
5426 level++;
5427 sd = sd->parent;
33859f7f 5428 if (!sd)
4dcf6aff
IM
5429 break;
5430 }
1da177e4 5431}
6d6bc0ad 5432#else /* !CONFIG_SCHED_DEBUG */
48f24c4d 5433# define sched_domain_debug(sd, cpu) do { } while (0)
d039ac60
PZ
5434static inline bool sched_debug(void)
5435{
5436 return false;
5437}
6d6bc0ad 5438#endif /* CONFIG_SCHED_DEBUG */
1da177e4 5439
1a20ff27 5440static int sd_degenerate(struct sched_domain *sd)
245af2c7 5441{
758b2cdc 5442 if (cpumask_weight(sched_domain_span(sd)) == 1)
245af2c7
SS
5443 return 1;
5444
5445 /* Following flags need at least 2 groups */
5446 if (sd->flags & (SD_LOAD_BALANCE |
5447 SD_BALANCE_NEWIDLE |
5448 SD_BALANCE_FORK |
89c4710e 5449 SD_BALANCE_EXEC |
5d4dfddd 5450 SD_SHARE_CPUCAPACITY |
d77b3ed5
VG
5451 SD_SHARE_PKG_RESOURCES |
5452 SD_SHARE_POWERDOMAIN)) {
245af2c7
SS
5453 if (sd->groups != sd->groups->next)
5454 return 0;
5455 }
5456
5457 /* Following flags don't use groups */
c88d5910 5458 if (sd->flags & (SD_WAKE_AFFINE))
245af2c7
SS
5459 return 0;
5460
5461 return 1;
5462}
5463
48f24c4d
IM
5464static int
5465sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
245af2c7
SS
5466{
5467 unsigned long cflags = sd->flags, pflags = parent->flags;
5468
5469 if (sd_degenerate(parent))
5470 return 1;
5471
758b2cdc 5472 if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
245af2c7
SS
5473 return 0;
5474
245af2c7
SS
5475 /* Flags needing groups don't count if only 1 group in parent */
5476 if (parent->groups == parent->groups->next) {
5477 pflags &= ~(SD_LOAD_BALANCE |
5478 SD_BALANCE_NEWIDLE |
5479 SD_BALANCE_FORK |
89c4710e 5480 SD_BALANCE_EXEC |
5d4dfddd 5481 SD_SHARE_CPUCAPACITY |
10866e62 5482 SD_SHARE_PKG_RESOURCES |
d77b3ed5
VG
5483 SD_PREFER_SIBLING |
5484 SD_SHARE_POWERDOMAIN);
5436499e
KC
5485 if (nr_node_ids == 1)
5486 pflags &= ~SD_SERIALIZE;
245af2c7
SS
5487 }
5488 if (~cflags & pflags)
5489 return 0;
5490
5491 return 1;
5492}
5493
dce840a0 5494static void free_rootdomain(struct rcu_head *rcu)
c6c4927b 5495{
dce840a0 5496 struct root_domain *rd = container_of(rcu, struct root_domain, rcu);
047106ad 5497
68e74568 5498 cpupri_cleanup(&rd->cpupri);
6bfd6d72 5499 cpudl_cleanup(&rd->cpudl);
1baca4ce 5500 free_cpumask_var(rd->dlo_mask);
c6c4927b
RR
5501 free_cpumask_var(rd->rto_mask);
5502 free_cpumask_var(rd->online);
5503 free_cpumask_var(rd->span);
5504 kfree(rd);
5505}
5506
57d885fe
GH
5507static void rq_attach_root(struct rq *rq, struct root_domain *rd)
5508{
a0490fa3 5509 struct root_domain *old_rd = NULL;
57d885fe 5510 unsigned long flags;
57d885fe 5511
05fa785c 5512 raw_spin_lock_irqsave(&rq->lock, flags);
57d885fe
GH
5513
5514 if (rq->rd) {
a0490fa3 5515 old_rd = rq->rd;
57d885fe 5516
c6c4927b 5517 if (cpumask_test_cpu(rq->cpu, old_rd->online))
1f11eb6a 5518 set_rq_offline(rq);
57d885fe 5519
c6c4927b 5520 cpumask_clear_cpu(rq->cpu, old_rd->span);
dc938520 5521
a0490fa3 5522 /*
0515973f 5523 * If we dont want to free the old_rd yet then
a0490fa3
IM
5524 * set old_rd to NULL to skip the freeing later
5525 * in this function:
5526 */
5527 if (!atomic_dec_and_test(&old_rd->refcount))
5528 old_rd = NULL;
57d885fe
GH
5529 }
5530
5531 atomic_inc(&rd->refcount);
5532 rq->rd = rd;
5533
c6c4927b 5534 cpumask_set_cpu(rq->cpu, rd->span);
00aec93d 5535 if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
1f11eb6a 5536 set_rq_online(rq);
57d885fe 5537
05fa785c 5538 raw_spin_unlock_irqrestore(&rq->lock, flags);
a0490fa3
IM
5539
5540 if (old_rd)
dce840a0 5541 call_rcu_sched(&old_rd->rcu, free_rootdomain);
57d885fe
GH
5542}
5543
68c38fc3 5544static int init_rootdomain(struct root_domain *rd)
57d885fe
GH
5545{
5546 memset(rd, 0, sizeof(*rd));
5547
68c38fc3 5548 if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
0c910d28 5549 goto out;
68c38fc3 5550 if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
c6c4927b 5551 goto free_span;
1baca4ce 5552 if (!alloc_cpumask_var(&rd->dlo_mask, GFP_KERNEL))
c6c4927b 5553 goto free_online;
1baca4ce
JL
5554 if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
5555 goto free_dlo_mask;
6e0534f2 5556
332ac17e 5557 init_dl_bw(&rd->dl_bw);
6bfd6d72
JL
5558 if (cpudl_init(&rd->cpudl) != 0)
5559 goto free_dlo_mask;
332ac17e 5560
68c38fc3 5561 if (cpupri_init(&rd->cpupri) != 0)
68e74568 5562 goto free_rto_mask;
c6c4927b 5563 return 0;
6e0534f2 5564
68e74568
RR
5565free_rto_mask:
5566 free_cpumask_var(rd->rto_mask);
1baca4ce
JL
5567free_dlo_mask:
5568 free_cpumask_var(rd->dlo_mask);
c6c4927b
RR
5569free_online:
5570 free_cpumask_var(rd->online);
5571free_span:
5572 free_cpumask_var(rd->span);
0c910d28 5573out:
c6c4927b 5574 return -ENOMEM;
57d885fe
GH
5575}
5576
029632fb
PZ
5577/*
5578 * By default the system creates a single root-domain with all cpus as
5579 * members (mimicking the global state we have today).
5580 */
5581struct root_domain def_root_domain;
5582
57d885fe
GH
5583static void init_defrootdomain(void)
5584{
68c38fc3 5585 init_rootdomain(&def_root_domain);
c6c4927b 5586
57d885fe
GH
5587 atomic_set(&def_root_domain.refcount, 1);
5588}
5589
dc938520 5590static struct root_domain *alloc_rootdomain(void)
57d885fe
GH
5591{
5592 struct root_domain *rd;
5593
5594 rd = kmalloc(sizeof(*rd), GFP_KERNEL);
5595 if (!rd)
5596 return NULL;
5597
68c38fc3 5598 if (init_rootdomain(rd) != 0) {
c6c4927b
RR
5599 kfree(rd);
5600 return NULL;
5601 }
57d885fe
GH
5602
5603 return rd;
5604}
5605
63b2ca30 5606static void free_sched_groups(struct sched_group *sg, int free_sgc)
e3589f6c
PZ
5607{
5608 struct sched_group *tmp, *first;
5609
5610 if (!sg)
5611 return;
5612
5613 first = sg;
5614 do {
5615 tmp = sg->next;
5616
63b2ca30
NP
5617 if (free_sgc && atomic_dec_and_test(&sg->sgc->ref))
5618 kfree(sg->sgc);
e3589f6c
PZ
5619
5620 kfree(sg);
5621 sg = tmp;
5622 } while (sg != first);
5623}
5624
dce840a0
PZ
5625static void free_sched_domain(struct rcu_head *rcu)
5626{
5627 struct sched_domain *sd = container_of(rcu, struct sched_domain, rcu);
e3589f6c
PZ
5628
5629 /*
5630 * If its an overlapping domain it has private groups, iterate and
5631 * nuke them all.
5632 */
5633 if (sd->flags & SD_OVERLAP) {
5634 free_sched_groups(sd->groups, 1);
5635 } else if (atomic_dec_and_test(&sd->groups->ref)) {
63b2ca30 5636 kfree(sd->groups->sgc);
dce840a0 5637 kfree(sd->groups);
9c3f75cb 5638 }
dce840a0
PZ
5639 kfree(sd);
5640}
5641
5642static void destroy_sched_domain(struct sched_domain *sd, int cpu)
5643{
5644 call_rcu(&sd->rcu, free_sched_domain);
5645}
5646
5647static void destroy_sched_domains(struct sched_domain *sd, int cpu)
5648{
5649 for (; sd; sd = sd->parent)
5650 destroy_sched_domain(sd, cpu);
5651}
5652
518cd623
PZ
5653/*
5654 * Keep a special pointer to the highest sched_domain that has
5655 * SD_SHARE_PKG_RESOURCE set (Last Level Cache Domain) for this
5656 * allows us to avoid some pointer chasing select_idle_sibling().
5657 *
5658 * Also keep a unique ID per domain (we use the first cpu number in
5659 * the cpumask of the domain), this allows us to quickly tell if
39be3501 5660 * two cpus are in the same cache domain, see cpus_share_cache().
518cd623
PZ
5661 */
5662DEFINE_PER_CPU(struct sched_domain *, sd_llc);
7d9ffa89 5663DEFINE_PER_CPU(int, sd_llc_size);
518cd623 5664DEFINE_PER_CPU(int, sd_llc_id);
fb13c7ee 5665DEFINE_PER_CPU(struct sched_domain *, sd_numa);
37dc6b50
PM
5666DEFINE_PER_CPU(struct sched_domain *, sd_busy);
5667DEFINE_PER_CPU(struct sched_domain *, sd_asym);
518cd623
PZ
5668
5669static void update_top_cache_domain(int cpu)
5670{
5671 struct sched_domain *sd;
5d4cf996 5672 struct sched_domain *busy_sd = NULL;
518cd623 5673 int id = cpu;
7d9ffa89 5674 int size = 1;
518cd623
PZ
5675
5676 sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES);
7d9ffa89 5677 if (sd) {
518cd623 5678 id = cpumask_first(sched_domain_span(sd));
7d9ffa89 5679 size = cpumask_weight(sched_domain_span(sd));
5d4cf996 5680 busy_sd = sd->parent; /* sd_busy */
7d9ffa89 5681 }
5d4cf996 5682 rcu_assign_pointer(per_cpu(sd_busy, cpu), busy_sd);
518cd623
PZ
5683
5684 rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
7d9ffa89 5685 per_cpu(sd_llc_size, cpu) = size;
518cd623 5686 per_cpu(sd_llc_id, cpu) = id;
fb13c7ee
MG
5687
5688 sd = lowest_flag_domain(cpu, SD_NUMA);
5689 rcu_assign_pointer(per_cpu(sd_numa, cpu), sd);
37dc6b50
PM
5690
5691 sd = highest_flag_domain(cpu, SD_ASYM_PACKING);
5692 rcu_assign_pointer(per_cpu(sd_asym, cpu), sd);
518cd623
PZ
5693}
5694
1da177e4 5695/*
0eab9146 5696 * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
1da177e4
LT
5697 * hold the hotplug lock.
5698 */
0eab9146
IM
5699static void
5700cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
1da177e4 5701{
70b97a7f 5702 struct rq *rq = cpu_rq(cpu);
245af2c7
SS
5703 struct sched_domain *tmp;
5704
5705 /* Remove the sched domains which do not contribute to scheduling. */
f29c9b1c 5706 for (tmp = sd; tmp; ) {
245af2c7
SS
5707 struct sched_domain *parent = tmp->parent;
5708 if (!parent)
5709 break;
f29c9b1c 5710
1a848870 5711 if (sd_parent_degenerate(tmp, parent)) {
245af2c7 5712 tmp->parent = parent->parent;
1a848870
SS
5713 if (parent->parent)
5714 parent->parent->child = tmp;
10866e62
PZ
5715 /*
5716 * Transfer SD_PREFER_SIBLING down in case of a
5717 * degenerate parent; the spans match for this
5718 * so the property transfers.
5719 */
5720 if (parent->flags & SD_PREFER_SIBLING)
5721 tmp->flags |= SD_PREFER_SIBLING;
dce840a0 5722 destroy_sched_domain(parent, cpu);
f29c9b1c
LZ
5723 } else
5724 tmp = tmp->parent;
245af2c7
SS
5725 }
5726
1a848870 5727 if (sd && sd_degenerate(sd)) {
dce840a0 5728 tmp = sd;
245af2c7 5729 sd = sd->parent;
dce840a0 5730 destroy_sched_domain(tmp, cpu);
1a848870
SS
5731 if (sd)
5732 sd->child = NULL;
5733 }
1da177e4 5734
4cb98839 5735 sched_domain_debug(sd, cpu);
1da177e4 5736
57d885fe 5737 rq_attach_root(rq, rd);
dce840a0 5738 tmp = rq->sd;
674311d5 5739 rcu_assign_pointer(rq->sd, sd);
dce840a0 5740 destroy_sched_domains(tmp, cpu);
518cd623
PZ
5741
5742 update_top_cache_domain(cpu);
1da177e4
LT
5743}
5744
5745/* cpus with isolated domains */
dcc30a35 5746static cpumask_var_t cpu_isolated_map;
1da177e4
LT
5747
5748/* Setup the mask of cpus configured for isolated domains */
5749static int __init isolated_cpu_setup(char *str)
5750{
bdddd296 5751 alloc_bootmem_cpumask_var(&cpu_isolated_map);
968ea6d8 5752 cpulist_parse(str, cpu_isolated_map);
1da177e4
LT
5753 return 1;
5754}
5755
8927f494 5756__setup("isolcpus=", isolated_cpu_setup);
1da177e4 5757
49a02c51 5758struct s_data {
21d42ccf 5759 struct sched_domain ** __percpu sd;
49a02c51
AH
5760 struct root_domain *rd;
5761};
5762
2109b99e 5763enum s_alloc {
2109b99e 5764 sa_rootdomain,
21d42ccf 5765 sa_sd,
dce840a0 5766 sa_sd_storage,
2109b99e
AH
5767 sa_none,
5768};
5769
c1174876
PZ
5770/*
5771 * Build an iteration mask that can exclude certain CPUs from the upwards
5772 * domain traversal.
5773 *
5774 * Asymmetric node setups can result in situations where the domain tree is of
5775 * unequal depth, make sure to skip domains that already cover the entire
5776 * range.
5777 *
5778 * In that case build_sched_domains() will have terminated the iteration early
5779 * and our sibling sd spans will be empty. Domains should always include the
5780 * cpu they're built on, so check that.
5781 *
5782 */
5783static void build_group_mask(struct sched_domain *sd, struct sched_group *sg)
5784{
5785 const struct cpumask *span = sched_domain_span(sd);
5786 struct sd_data *sdd = sd->private;
5787 struct sched_domain *sibling;
5788 int i;
5789
5790 for_each_cpu(i, span) {
5791 sibling = *per_cpu_ptr(sdd->sd, i);
5792 if (!cpumask_test_cpu(i, sched_domain_span(sibling)))
5793 continue;
5794
5795 cpumask_set_cpu(i, sched_group_mask(sg));
5796 }
5797}
5798
5799/*
5800 * Return the canonical balance cpu for this group, this is the first cpu
5801 * of this group that's also in the iteration mask.
5802 */
5803int group_balance_cpu(struct sched_group *sg)
5804{
5805 return cpumask_first_and(sched_group_cpus(sg), sched_group_mask(sg));
5806}
5807
e3589f6c
PZ
5808static int
5809build_overlap_sched_groups(struct sched_domain *sd, int cpu)
5810{
5811 struct sched_group *first = NULL, *last = NULL, *groups = NULL, *sg;
5812 const struct cpumask *span = sched_domain_span(sd);
5813 struct cpumask *covered = sched_domains_tmpmask;
5814 struct sd_data *sdd = sd->private;
aaecac4a 5815 struct sched_domain *sibling;
e3589f6c
PZ
5816 int i;
5817
5818 cpumask_clear(covered);
5819
5820 for_each_cpu(i, span) {
5821 struct cpumask *sg_span;
5822
5823 if (cpumask_test_cpu(i, covered))
5824 continue;
5825
aaecac4a 5826 sibling = *per_cpu_ptr(sdd->sd, i);
c1174876
PZ
5827
5828 /* See the comment near build_group_mask(). */
aaecac4a 5829 if (!cpumask_test_cpu(i, sched_domain_span(sibling)))
c1174876
PZ
5830 continue;
5831
e3589f6c 5832 sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
4d78a223 5833 GFP_KERNEL, cpu_to_node(cpu));
e3589f6c
PZ
5834
5835 if (!sg)
5836 goto fail;
5837
5838 sg_span = sched_group_cpus(sg);
aaecac4a
ZZ
5839 if (sibling->child)
5840 cpumask_copy(sg_span, sched_domain_span(sibling->child));
5841 else
e3589f6c
PZ
5842 cpumask_set_cpu(i, sg_span);
5843
5844 cpumask_or(covered, covered, sg_span);
5845
63b2ca30
NP
5846 sg->sgc = *per_cpu_ptr(sdd->sgc, i);
5847 if (atomic_inc_return(&sg->sgc->ref) == 1)
c1174876
PZ
5848 build_group_mask(sd, sg);
5849
c3decf0d 5850 /*
63b2ca30 5851 * Initialize sgc->capacity such that even if we mess up the
c3decf0d
PZ
5852 * domains and no possible iteration will get us here, we won't
5853 * die on a /0 trap.
5854 */
ca8ce3d0 5855 sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span);
63b2ca30 5856 sg->sgc->capacity_orig = sg->sgc->capacity;
e3589f6c 5857
c1174876
PZ
5858 /*
5859 * Make sure the first group of this domain contains the
5860 * canonical balance cpu. Otherwise the sched_domain iteration
5861 * breaks. See update_sg_lb_stats().
5862 */
74a5ce20 5863 if ((!groups && cpumask_test_cpu(cpu, sg_span)) ||
c1174876 5864 group_balance_cpu(sg) == cpu)
e3589f6c
PZ
5865 groups = sg;
5866
5867 if (!first)
5868 first = sg;
5869 if (last)
5870 last->next = sg;
5871 last = sg;
5872 last->next = first;
5873 }
5874 sd->groups = groups;
5875
5876 return 0;
5877
5878fail:
5879 free_sched_groups(first, 0);
5880
5881 return -ENOMEM;
5882}
5883
dce840a0 5884static int get_group(int cpu, struct sd_data *sdd, struct sched_group **sg)
1da177e4 5885{
dce840a0
PZ
5886 struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);
5887 struct sched_domain *child = sd->child;
1da177e4 5888
dce840a0
PZ
5889 if (child)
5890 cpu = cpumask_first(sched_domain_span(child));
1e9f28fa 5891
9c3f75cb 5892 if (sg) {
dce840a0 5893 *sg = *per_cpu_ptr(sdd->sg, cpu);
63b2ca30
NP
5894 (*sg)->sgc = *per_cpu_ptr(sdd->sgc, cpu);
5895 atomic_set(&(*sg)->sgc->ref, 1); /* for claim_allocations */
9c3f75cb 5896 }
dce840a0
PZ
5897
5898 return cpu;
1e9f28fa 5899}
1e9f28fa 5900
01a08546 5901/*
dce840a0
PZ
5902 * build_sched_groups will build a circular linked list of the groups
5903 * covered by the given span, and will set each group's ->cpumask correctly,
ced549fa 5904 * and ->cpu_capacity to 0.
e3589f6c
PZ
5905 *
5906 * Assumes the sched_domain tree is fully constructed
01a08546 5907 */
e3589f6c
PZ
5908static int
5909build_sched_groups(struct sched_domain *sd, int cpu)
1da177e4 5910{
dce840a0
PZ
5911 struct sched_group *first = NULL, *last = NULL;
5912 struct sd_data *sdd = sd->private;
5913 const struct cpumask *span = sched_domain_span(sd);
f96225fd 5914 struct cpumask *covered;
dce840a0 5915 int i;
9c1cfda2 5916
e3589f6c
PZ
5917 get_group(cpu, sdd, &sd->groups);
5918 atomic_inc(&sd->groups->ref);
5919
0936629f 5920 if (cpu != cpumask_first(span))
e3589f6c
PZ
5921 return 0;
5922
f96225fd
PZ
5923 lockdep_assert_held(&sched_domains_mutex);
5924 covered = sched_domains_tmpmask;
5925
dce840a0 5926 cpumask_clear(covered);
6711cab4 5927
dce840a0
PZ
5928 for_each_cpu(i, span) {
5929 struct sched_group *sg;
cd08e923 5930 int group, j;
6711cab4 5931
dce840a0
PZ
5932 if (cpumask_test_cpu(i, covered))
5933 continue;
6711cab4 5934
cd08e923 5935 group = get_group(i, sdd, &sg);
c1174876 5936 cpumask_setall(sched_group_mask(sg));
0601a88d 5937
dce840a0
PZ
5938 for_each_cpu(j, span) {
5939 if (get_group(j, sdd, NULL) != group)
5940 continue;
0601a88d 5941
dce840a0
PZ
5942 cpumask_set_cpu(j, covered);
5943 cpumask_set_cpu(j, sched_group_cpus(sg));
5944 }
0601a88d 5945
dce840a0
PZ
5946 if (!first)
5947 first = sg;
5948 if (last)
5949 last->next = sg;
5950 last = sg;
5951 }
5952 last->next = first;
e3589f6c
PZ
5953
5954 return 0;
0601a88d 5955}
51888ca2 5956
89c4710e 5957/*
63b2ca30 5958 * Initialize sched groups cpu_capacity.
89c4710e 5959 *
63b2ca30 5960 * cpu_capacity indicates the capacity of sched group, which is used while
89c4710e 5961 * distributing the load between different sched groups in a sched domain.
63b2ca30
NP
5962 * Typically cpu_capacity for all the groups in a sched domain will be same
5963 * unless there are asymmetries in the topology. If there are asymmetries,
5964 * group having more cpu_capacity will pickup more load compared to the
5965 * group having less cpu_capacity.
89c4710e 5966 */
63b2ca30 5967static void init_sched_groups_capacity(int cpu, struct sched_domain *sd)
89c4710e 5968{
e3589f6c 5969 struct sched_group *sg = sd->groups;
89c4710e 5970
94c95ba6 5971 WARN_ON(!sg);
e3589f6c
PZ
5972
5973 do {
5974 sg->group_weight = cpumask_weight(sched_group_cpus(sg));
5975 sg = sg->next;
5976 } while (sg != sd->groups);
89c4710e 5977
c1174876 5978 if (cpu != group_balance_cpu(sg))
e3589f6c 5979 return;
aae6d3dd 5980
63b2ca30
NP
5981 update_group_capacity(sd, cpu);
5982 atomic_set(&sg->sgc->nr_busy_cpus, sg->group_weight);
89c4710e
SS
5983}
5984
7c16ec58
MT
5985/*
5986 * Initializers for schedule domains
5987 * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
5988 */
5989
1d3504fc 5990static int default_relax_domain_level = -1;
60495e77 5991int sched_domain_level_max;
1d3504fc
HS
5992
5993static int __init setup_relax_domain_level(char *str)
5994{
a841f8ce
DS
5995 if (kstrtoint(str, 0, &default_relax_domain_level))
5996 pr_warn("Unable to set relax_domain_level\n");
30e0e178 5997
1d3504fc
HS
5998 return 1;
5999}
6000__setup("relax_domain_level=", setup_relax_domain_level);
6001
6002static void set_domain_attribute(struct sched_domain *sd,
6003 struct sched_domain_attr *attr)
6004{
6005 int request;
6006
6007 if (!attr || attr->relax_domain_level < 0) {
6008 if (default_relax_domain_level < 0)
6009 return;
6010 else
6011 request = default_relax_domain_level;
6012 } else
6013 request = attr->relax_domain_level;
6014 if (request < sd->level) {
6015 /* turn off idle balance on this domain */
c88d5910 6016 sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
1d3504fc
HS
6017 } else {
6018 /* turn on idle balance on this domain */
c88d5910 6019 sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
1d3504fc
HS
6020 }
6021}
6022
54ab4ff4
PZ
6023static void __sdt_free(const struct cpumask *cpu_map);
6024static int __sdt_alloc(const struct cpumask *cpu_map);
6025
2109b99e
AH
6026static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
6027 const struct cpumask *cpu_map)
6028{
6029 switch (what) {
2109b99e 6030 case sa_rootdomain:
822ff793
PZ
6031 if (!atomic_read(&d->rd->refcount))
6032 free_rootdomain(&d->rd->rcu); /* fall through */
21d42ccf
PZ
6033 case sa_sd:
6034 free_percpu(d->sd); /* fall through */
dce840a0 6035 case sa_sd_storage:
54ab4ff4 6036 __sdt_free(cpu_map); /* fall through */
2109b99e
AH
6037 case sa_none:
6038 break;
6039 }
6040}
3404c8d9 6041
2109b99e
AH
6042static enum s_alloc __visit_domain_allocation_hell(struct s_data *d,
6043 const struct cpumask *cpu_map)
6044{
dce840a0
PZ
6045 memset(d, 0, sizeof(*d));
6046
54ab4ff4
PZ
6047 if (__sdt_alloc(cpu_map))
6048 return sa_sd_storage;
dce840a0
PZ
6049 d->sd = alloc_percpu(struct sched_domain *);
6050 if (!d->sd)
6051 return sa_sd_storage;
2109b99e 6052 d->rd = alloc_rootdomain();
dce840a0 6053 if (!d->rd)
21d42ccf 6054 return sa_sd;
2109b99e
AH
6055 return sa_rootdomain;
6056}
57d885fe 6057
dce840a0
PZ
6058/*
6059 * NULL the sd_data elements we've used to build the sched_domain and
6060 * sched_group structure so that the subsequent __free_domain_allocs()
6061 * will not free the data we're using.
6062 */
6063static void claim_allocations(int cpu, struct sched_domain *sd)
6064{
6065 struct sd_data *sdd = sd->private;
dce840a0
PZ
6066
6067 WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd);
6068 *per_cpu_ptr(sdd->sd, cpu) = NULL;
6069
e3589f6c 6070 if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref))
dce840a0 6071 *per_cpu_ptr(sdd->sg, cpu) = NULL;
e3589f6c 6072
63b2ca30
NP
6073 if (atomic_read(&(*per_cpu_ptr(sdd->sgc, cpu))->ref))
6074 *per_cpu_ptr(sdd->sgc, cpu) = NULL;
dce840a0
PZ
6075}
6076
cb83b629 6077#ifdef CONFIG_NUMA
cb83b629 6078static int sched_domains_numa_levels;
cb83b629
PZ
6079static int *sched_domains_numa_distance;
6080static struct cpumask ***sched_domains_numa_masks;
6081static int sched_domains_curr_level;
143e1e28 6082#endif
cb83b629 6083
143e1e28
VG
6084/*
6085 * SD_flags allowed in topology descriptions.
6086 *
5d4dfddd 6087 * SD_SHARE_CPUCAPACITY - describes SMT topologies
143e1e28
VG
6088 * SD_SHARE_PKG_RESOURCES - describes shared caches
6089 * SD_NUMA - describes NUMA topologies
d77b3ed5 6090 * SD_SHARE_POWERDOMAIN - describes shared power domain
143e1e28
VG
6091 *
6092 * Odd one out:
6093 * SD_ASYM_PACKING - describes SMT quirks
6094 */
6095#define TOPOLOGY_SD_FLAGS \
5d4dfddd 6096 (SD_SHARE_CPUCAPACITY | \
143e1e28
VG
6097 SD_SHARE_PKG_RESOURCES | \
6098 SD_NUMA | \
d77b3ed5
VG
6099 SD_ASYM_PACKING | \
6100 SD_SHARE_POWERDOMAIN)
cb83b629
PZ
6101
6102static struct sched_domain *
143e1e28 6103sd_init(struct sched_domain_topology_level *tl, int cpu)
cb83b629
PZ
6104{
6105 struct sched_domain *sd = *per_cpu_ptr(tl->data.sd, cpu);
143e1e28
VG
6106 int sd_weight, sd_flags = 0;
6107
6108#ifdef CONFIG_NUMA
6109 /*
6110 * Ugly hack to pass state to sd_numa_mask()...
6111 */
6112 sched_domains_curr_level = tl->numa_level;
6113#endif
6114
6115 sd_weight = cpumask_weight(tl->mask(cpu));
6116
6117 if (tl->sd_flags)
6118 sd_flags = (*tl->sd_flags)();
6119 if (WARN_ONCE(sd_flags & ~TOPOLOGY_SD_FLAGS,
6120 "wrong sd_flags in topology description\n"))
6121 sd_flags &= ~TOPOLOGY_SD_FLAGS;
cb83b629
PZ
6122
6123 *sd = (struct sched_domain){
6124 .min_interval = sd_weight,
6125 .max_interval = 2*sd_weight,
6126 .busy_factor = 32,
870a0bb5 6127 .imbalance_pct = 125,
143e1e28
VG
6128
6129 .cache_nice_tries = 0,
6130 .busy_idx = 0,
6131 .idle_idx = 0,
cb83b629
PZ
6132 .newidle_idx = 0,
6133 .wake_idx = 0,
6134 .forkexec_idx = 0,
6135
6136 .flags = 1*SD_LOAD_BALANCE
6137 | 1*SD_BALANCE_NEWIDLE
143e1e28
VG
6138 | 1*SD_BALANCE_EXEC
6139 | 1*SD_BALANCE_FORK
cb83b629 6140 | 0*SD_BALANCE_WAKE
143e1e28 6141 | 1*SD_WAKE_AFFINE
5d4dfddd 6142 | 0*SD_SHARE_CPUCAPACITY
cb83b629 6143 | 0*SD_SHARE_PKG_RESOURCES
143e1e28 6144 | 0*SD_SERIALIZE
cb83b629 6145 | 0*SD_PREFER_SIBLING
143e1e28
VG
6146 | 0*SD_NUMA
6147 | sd_flags
cb83b629 6148 ,
143e1e28 6149
cb83b629
PZ
6150 .last_balance = jiffies,
6151 .balance_interval = sd_weight,
143e1e28 6152 .smt_gain = 0,
2b4cfe64
JL
6153 .max_newidle_lb_cost = 0,
6154 .next_decay_max_lb_cost = jiffies,
143e1e28
VG
6155#ifdef CONFIG_SCHED_DEBUG
6156 .name = tl->name,
6157#endif
cb83b629 6158 };
cb83b629
PZ
6159
6160 /*
143e1e28 6161 * Convert topological properties into behaviour.
cb83b629 6162 */
143e1e28 6163
5d4dfddd 6164 if (sd->flags & SD_SHARE_CPUCAPACITY) {
143e1e28
VG
6165 sd->imbalance_pct = 110;
6166 sd->smt_gain = 1178; /* ~15% */
143e1e28
VG
6167
6168 } else if (sd->flags & SD_SHARE_PKG_RESOURCES) {
6169 sd->imbalance_pct = 117;
6170 sd->cache_nice_tries = 1;
6171 sd->busy_idx = 2;
6172
6173#ifdef CONFIG_NUMA
6174 } else if (sd->flags & SD_NUMA) {
6175 sd->cache_nice_tries = 2;
6176 sd->busy_idx = 3;
6177 sd->idle_idx = 2;
6178
6179 sd->flags |= SD_SERIALIZE;
6180 if (sched_domains_numa_distance[tl->numa_level] > RECLAIM_DISTANCE) {
6181 sd->flags &= ~(SD_BALANCE_EXEC |
6182 SD_BALANCE_FORK |
6183 SD_WAKE_AFFINE);
6184 }
6185
6186#endif
6187 } else {
6188 sd->flags |= SD_PREFER_SIBLING;
6189 sd->cache_nice_tries = 1;
6190 sd->busy_idx = 2;
6191 sd->idle_idx = 1;
6192 }
6193
6194 sd->private = &tl->data;
cb83b629
PZ
6195
6196 return sd;
6197}
6198
143e1e28
VG
6199/*
6200 * Topology list, bottom-up.
6201 */
6202static struct sched_domain_topology_level default_topology[] = {
6203#ifdef CONFIG_SCHED_SMT
6204 { cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) },
6205#endif
6206#ifdef CONFIG_SCHED_MC
6207 { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) },
143e1e28
VG
6208#endif
6209 { cpu_cpu_mask, SD_INIT_NAME(DIE) },
6210 { NULL, },
6211};
6212
6213struct sched_domain_topology_level *sched_domain_topology = default_topology;
6214
6215#define for_each_sd_topology(tl) \
6216 for (tl = sched_domain_topology; tl->mask; tl++)
6217
6218void set_sched_topology(struct sched_domain_topology_level *tl)
6219{
6220 sched_domain_topology = tl;
6221}
6222
6223#ifdef CONFIG_NUMA
6224
cb83b629
PZ
6225static const struct cpumask *sd_numa_mask(int cpu)
6226{
6227 return sched_domains_numa_masks[sched_domains_curr_level][cpu_to_node(cpu)];
6228}
6229
d039ac60
PZ
6230static void sched_numa_warn(const char *str)
6231{
6232 static int done = false;
6233 int i,j;
6234
6235 if (done)
6236 return;
6237
6238 done = true;
6239
6240 printk(KERN_WARNING "ERROR: %s\n\n", str);
6241
6242 for (i = 0; i < nr_node_ids; i++) {
6243 printk(KERN_WARNING " ");
6244 for (j = 0; j < nr_node_ids; j++)
6245 printk(KERN_CONT "%02d ", node_distance(i,j));
6246 printk(KERN_CONT "\n");
6247 }
6248 printk(KERN_WARNING "\n");
6249}
6250
6251static bool find_numa_distance(int distance)
6252{
6253 int i;
6254
6255 if (distance == node_distance(0, 0))
6256 return true;
6257
6258 for (i = 0; i < sched_domains_numa_levels; i++) {
6259 if (sched_domains_numa_distance[i] == distance)
6260 return true;
6261 }
6262
6263 return false;
6264}
6265
cb83b629
PZ
6266static void sched_init_numa(void)
6267{
6268 int next_distance, curr_distance = node_distance(0, 0);
6269 struct sched_domain_topology_level *tl;
6270 int level = 0;
6271 int i, j, k;
6272
cb83b629
PZ
6273 sched_domains_numa_distance = kzalloc(sizeof(int) * nr_node_ids, GFP_KERNEL);
6274 if (!sched_domains_numa_distance)
6275 return;
6276
6277 /*
6278 * O(nr_nodes^2) deduplicating selection sort -- in order to find the
6279 * unique distances in the node_distance() table.
6280 *
6281 * Assumes node_distance(0,j) includes all distances in
6282 * node_distance(i,j) in order to avoid cubic time.
cb83b629
PZ
6283 */
6284 next_distance = curr_distance;
6285 for (i = 0; i < nr_node_ids; i++) {
6286 for (j = 0; j < nr_node_ids; j++) {
d039ac60
PZ
6287 for (k = 0; k < nr_node_ids; k++) {
6288 int distance = node_distance(i, k);
6289
6290 if (distance > curr_distance &&
6291 (distance < next_distance ||
6292 next_distance == curr_distance))
6293 next_distance = distance;
6294
6295 /*
6296 * While not a strong assumption it would be nice to know
6297 * about cases where if node A is connected to B, B is not
6298 * equally connected to A.
6299 */
6300 if (sched_debug() && node_distance(k, i) != distance)
6301 sched_numa_warn("Node-distance not symmetric");
6302
6303 if (sched_debug() && i && !find_numa_distance(distance))
6304 sched_numa_warn("Node-0 not representative");
6305 }
6306 if (next_distance != curr_distance) {
6307 sched_domains_numa_distance[level++] = next_distance;
6308 sched_domains_numa_levels = level;
6309 curr_distance = next_distance;
6310 } else break;
cb83b629 6311 }
d039ac60
PZ
6312
6313 /*
6314 * In case of sched_debug() we verify the above assumption.
6315 */
6316 if (!sched_debug())
6317 break;
cb83b629
PZ
6318 }
6319 /*
6320 * 'level' contains the number of unique distances, excluding the
6321 * identity distance node_distance(i,i).
6322 *
28b4a521 6323 * The sched_domains_numa_distance[] array includes the actual distance
cb83b629
PZ
6324 * numbers.
6325 */
6326
5f7865f3
TC
6327 /*
6328 * Here, we should temporarily reset sched_domains_numa_levels to 0.
6329 * If it fails to allocate memory for array sched_domains_numa_masks[][],
6330 * the array will contain less then 'level' members. This could be
6331 * dangerous when we use it to iterate array sched_domains_numa_masks[][]
6332 * in other functions.
6333 *
6334 * We reset it to 'level' at the end of this function.
6335 */
6336 sched_domains_numa_levels = 0;
6337
cb83b629
PZ
6338 sched_domains_numa_masks = kzalloc(sizeof(void *) * level, GFP_KERNEL);
6339 if (!sched_domains_numa_masks)
6340 return;
6341
6342 /*
6343 * Now for each level, construct a mask per node which contains all
6344 * cpus of nodes that are that many hops away from us.
6345 */
6346 for (i = 0; i < level; i++) {
6347 sched_domains_numa_masks[i] =
6348 kzalloc(nr_node_ids * sizeof(void *), GFP_KERNEL);
6349 if (!sched_domains_numa_masks[i])
6350 return;
6351
6352 for (j = 0; j < nr_node_ids; j++) {
2ea45800 6353 struct cpumask *mask = kzalloc(cpumask_size(), GFP_KERNEL);
cb83b629
PZ
6354 if (!mask)
6355 return;
6356
6357 sched_domains_numa_masks[i][j] = mask;
6358
6359 for (k = 0; k < nr_node_ids; k++) {
dd7d8634 6360 if (node_distance(j, k) > sched_domains_numa_distance[i])
cb83b629
PZ
6361 continue;
6362
6363 cpumask_or(mask, mask, cpumask_of_node(k));
6364 }
6365 }
6366 }
6367
143e1e28
VG
6368 /* Compute default topology size */
6369 for (i = 0; sched_domain_topology[i].mask; i++);
6370
c515db8c 6371 tl = kzalloc((i + level + 1) *
cb83b629
PZ
6372 sizeof(struct sched_domain_topology_level), GFP_KERNEL);
6373 if (!tl)
6374 return;
6375
6376 /*
6377 * Copy the default topology bits..
6378 */
143e1e28
VG
6379 for (i = 0; sched_domain_topology[i].mask; i++)
6380 tl[i] = sched_domain_topology[i];
cb83b629
PZ
6381
6382 /*
6383 * .. and append 'j' levels of NUMA goodness.
6384 */
6385 for (j = 0; j < level; i++, j++) {
6386 tl[i] = (struct sched_domain_topology_level){
cb83b629 6387 .mask = sd_numa_mask,
143e1e28 6388 .sd_flags = cpu_numa_flags,
cb83b629
PZ
6389 .flags = SDTL_OVERLAP,
6390 .numa_level = j,
143e1e28 6391 SD_INIT_NAME(NUMA)
cb83b629
PZ
6392 };
6393 }
6394
6395 sched_domain_topology = tl;
5f7865f3
TC
6396
6397 sched_domains_numa_levels = level;
cb83b629 6398}
301a5cba
TC
6399
6400static void sched_domains_numa_masks_set(int cpu)
6401{
6402 int i, j;
6403 int node = cpu_to_node(cpu);
6404
6405 for (i = 0; i < sched_domains_numa_levels; i++) {
6406 for (j = 0; j < nr_node_ids; j++) {
6407 if (node_distance(j, node) <= sched_domains_numa_distance[i])
6408 cpumask_set_cpu(cpu, sched_domains_numa_masks[i][j]);
6409 }
6410 }
6411}
6412
6413static void sched_domains_numa_masks_clear(int cpu)
6414{
6415 int i, j;
6416 for (i = 0; i < sched_domains_numa_levels; i++) {
6417 for (j = 0; j < nr_node_ids; j++)
6418 cpumask_clear_cpu(cpu, sched_domains_numa_masks[i][j]);
6419 }
6420}
6421
6422/*
6423 * Update sched_domains_numa_masks[level][node] array when new cpus
6424 * are onlined.
6425 */
6426static int sched_domains_numa_masks_update(struct notifier_block *nfb,
6427 unsigned long action,
6428 void *hcpu)
6429{
6430 int cpu = (long)hcpu;
6431
6432 switch (action & ~CPU_TASKS_FROZEN) {
6433 case CPU_ONLINE:
6434 sched_domains_numa_masks_set(cpu);
6435 break;
6436
6437 case CPU_DEAD:
6438 sched_domains_numa_masks_clear(cpu);
6439 break;
6440
6441 default:
6442 return NOTIFY_DONE;
6443 }
6444
6445 return NOTIFY_OK;
cb83b629
PZ
6446}
6447#else
6448static inline void sched_init_numa(void)
6449{
6450}
301a5cba
TC
6451
6452static int sched_domains_numa_masks_update(struct notifier_block *nfb,
6453 unsigned long action,
6454 void *hcpu)
6455{
6456 return 0;
6457}
cb83b629
PZ
6458#endif /* CONFIG_NUMA */
6459
54ab4ff4
PZ
6460static int __sdt_alloc(const struct cpumask *cpu_map)
6461{
6462 struct sched_domain_topology_level *tl;
6463 int j;
6464
27723a68 6465 for_each_sd_topology(tl) {
54ab4ff4
PZ
6466 struct sd_data *sdd = &tl->data;
6467
6468 sdd->sd = alloc_percpu(struct sched_domain *);
6469 if (!sdd->sd)
6470 return -ENOMEM;
6471
6472 sdd->sg = alloc_percpu(struct sched_group *);
6473 if (!sdd->sg)
6474 return -ENOMEM;
6475
63b2ca30
NP
6476 sdd->sgc = alloc_percpu(struct sched_group_capacity *);
6477 if (!sdd->sgc)
9c3f75cb
PZ
6478 return -ENOMEM;
6479
54ab4ff4
PZ
6480 for_each_cpu(j, cpu_map) {
6481 struct sched_domain *sd;
6482 struct sched_group *sg;
63b2ca30 6483 struct sched_group_capacity *sgc;
54ab4ff4
PZ
6484
6485 sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(),
6486 GFP_KERNEL, cpu_to_node(j));
6487 if (!sd)
6488 return -ENOMEM;
6489
6490 *per_cpu_ptr(sdd->sd, j) = sd;
6491
6492 sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
6493 GFP_KERNEL, cpu_to_node(j));
6494 if (!sg)
6495 return -ENOMEM;
6496
30b4e9eb
IM
6497 sg->next = sg;
6498
54ab4ff4 6499 *per_cpu_ptr(sdd->sg, j) = sg;
9c3f75cb 6500
63b2ca30 6501 sgc = kzalloc_node(sizeof(struct sched_group_capacity) + cpumask_size(),
9c3f75cb 6502 GFP_KERNEL, cpu_to_node(j));
63b2ca30 6503 if (!sgc)
9c3f75cb
PZ
6504 return -ENOMEM;
6505
63b2ca30 6506 *per_cpu_ptr(sdd->sgc, j) = sgc;
54ab4ff4
PZ
6507 }
6508 }
6509
6510 return 0;
6511}
6512
6513static void __sdt_free(const struct cpumask *cpu_map)
6514{
6515 struct sched_domain_topology_level *tl;
6516 int j;
6517
27723a68 6518 for_each_sd_topology(tl) {
54ab4ff4
PZ
6519 struct sd_data *sdd = &tl->data;
6520
6521 for_each_cpu(j, cpu_map) {
fb2cf2c6 6522 struct sched_domain *sd;
6523
6524 if (sdd->sd) {
6525 sd = *per_cpu_ptr(sdd->sd, j);
6526 if (sd && (sd->flags & SD_OVERLAP))
6527 free_sched_groups(sd->groups, 0);
6528 kfree(*per_cpu_ptr(sdd->sd, j));
6529 }
6530
6531 if (sdd->sg)
6532 kfree(*per_cpu_ptr(sdd->sg, j));
63b2ca30
NP
6533 if (sdd->sgc)
6534 kfree(*per_cpu_ptr(sdd->sgc, j));
54ab4ff4
PZ
6535 }
6536 free_percpu(sdd->sd);
fb2cf2c6 6537 sdd->sd = NULL;
54ab4ff4 6538 free_percpu(sdd->sg);
fb2cf2c6 6539 sdd->sg = NULL;
63b2ca30
NP
6540 free_percpu(sdd->sgc);
6541 sdd->sgc = NULL;
54ab4ff4
PZ
6542 }
6543}
6544
2c402dc3 6545struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl,
4a850cbe
VK
6546 const struct cpumask *cpu_map, struct sched_domain_attr *attr,
6547 struct sched_domain *child, int cpu)
2c402dc3 6548{
143e1e28 6549 struct sched_domain *sd = sd_init(tl, cpu);
2c402dc3 6550 if (!sd)
d069b916 6551 return child;
2c402dc3 6552
2c402dc3 6553 cpumask_and(sched_domain_span(sd), cpu_map, tl->mask(cpu));
60495e77
PZ
6554 if (child) {
6555 sd->level = child->level + 1;
6556 sched_domain_level_max = max(sched_domain_level_max, sd->level);
d069b916 6557 child->parent = sd;
c75e0128 6558 sd->child = child;
6ae72dff
PZ
6559
6560 if (!cpumask_subset(sched_domain_span(child),
6561 sched_domain_span(sd))) {
6562 pr_err("BUG: arch topology borken\n");
6563#ifdef CONFIG_SCHED_DEBUG
6564 pr_err(" the %s domain not a subset of the %s domain\n",
6565 child->name, sd->name);
6566#endif
6567 /* Fixup, ensure @sd has at least @child cpus. */
6568 cpumask_or(sched_domain_span(sd),
6569 sched_domain_span(sd),
6570 sched_domain_span(child));
6571 }
6572
60495e77 6573 }
a841f8ce 6574 set_domain_attribute(sd, attr);
2c402dc3
PZ
6575
6576 return sd;
6577}
6578
2109b99e
AH
6579/*
6580 * Build sched domains for a given set of cpus and attach the sched domains
6581 * to the individual cpus
6582 */
dce840a0
PZ
6583static int build_sched_domains(const struct cpumask *cpu_map,
6584 struct sched_domain_attr *attr)
2109b99e 6585{
1c632169 6586 enum s_alloc alloc_state;
dce840a0 6587 struct sched_domain *sd;
2109b99e 6588 struct s_data d;
822ff793 6589 int i, ret = -ENOMEM;
9c1cfda2 6590
2109b99e
AH
6591 alloc_state = __visit_domain_allocation_hell(&d, cpu_map);
6592 if (alloc_state != sa_rootdomain)
6593 goto error;
9c1cfda2 6594
dce840a0 6595 /* Set up domains for cpus specified by the cpu_map. */
abcd083a 6596 for_each_cpu(i, cpu_map) {
eb7a74e6
PZ
6597 struct sched_domain_topology_level *tl;
6598
3bd65a80 6599 sd = NULL;
27723a68 6600 for_each_sd_topology(tl) {
4a850cbe 6601 sd = build_sched_domain(tl, cpu_map, attr, sd, i);
22da9569
VK
6602 if (tl == sched_domain_topology)
6603 *per_cpu_ptr(d.sd, i) = sd;
e3589f6c
PZ
6604 if (tl->flags & SDTL_OVERLAP || sched_feat(FORCE_SD_OVERLAP))
6605 sd->flags |= SD_OVERLAP;
d110235d
PZ
6606 if (cpumask_equal(cpu_map, sched_domain_span(sd)))
6607 break;
e3589f6c 6608 }
dce840a0
PZ
6609 }
6610
6611 /* Build the groups for the domains */
6612 for_each_cpu(i, cpu_map) {
6613 for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
6614 sd->span_weight = cpumask_weight(sched_domain_span(sd));
e3589f6c
PZ
6615 if (sd->flags & SD_OVERLAP) {
6616 if (build_overlap_sched_groups(sd, i))
6617 goto error;
6618 } else {
6619 if (build_sched_groups(sd, i))
6620 goto error;
6621 }
1cf51902 6622 }
a06dadbe 6623 }
9c1cfda2 6624
ced549fa 6625 /* Calculate CPU capacity for physical packages and nodes */
a9c9a9b6
PZ
6626 for (i = nr_cpumask_bits-1; i >= 0; i--) {
6627 if (!cpumask_test_cpu(i, cpu_map))
6628 continue;
9c1cfda2 6629
dce840a0
PZ
6630 for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
6631 claim_allocations(i, sd);
63b2ca30 6632 init_sched_groups_capacity(i, sd);
dce840a0 6633 }
f712c0c7 6634 }
9c1cfda2 6635
1da177e4 6636 /* Attach the domains */
dce840a0 6637 rcu_read_lock();
abcd083a 6638 for_each_cpu(i, cpu_map) {
21d42ccf 6639 sd = *per_cpu_ptr(d.sd, i);
49a02c51 6640 cpu_attach_domain(sd, d.rd, i);
1da177e4 6641 }
dce840a0 6642 rcu_read_unlock();
51888ca2 6643
822ff793 6644 ret = 0;
51888ca2 6645error:
2109b99e 6646 __free_domain_allocs(&d, alloc_state, cpu_map);
822ff793 6647 return ret;
1da177e4 6648}
029190c5 6649
acc3f5d7 6650static cpumask_var_t *doms_cur; /* current sched domains */
029190c5 6651static int ndoms_cur; /* number of sched domains in 'doms_cur' */
4285f594
IM
6652static struct sched_domain_attr *dattr_cur;
6653 /* attribues of custom domains in 'doms_cur' */
029190c5
PJ
6654
6655/*
6656 * Special case: If a kmalloc of a doms_cur partition (array of
4212823f
RR
6657 * cpumask) fails, then fallback to a single sched domain,
6658 * as determined by the single cpumask fallback_doms.
029190c5 6659 */
4212823f 6660static cpumask_var_t fallback_doms;
029190c5 6661
ee79d1bd
HC
6662/*
6663 * arch_update_cpu_topology lets virtualized architectures update the
6664 * cpu core maps. It is supposed to return 1 if the topology changed
6665 * or 0 if it stayed the same.
6666 */
52f5684c 6667int __weak arch_update_cpu_topology(void)
22e52b07 6668{
ee79d1bd 6669 return 0;
22e52b07
HC
6670}
6671
acc3f5d7
RR
6672cpumask_var_t *alloc_sched_domains(unsigned int ndoms)
6673{
6674 int i;
6675 cpumask_var_t *doms;
6676
6677 doms = kmalloc(sizeof(*doms) * ndoms, GFP_KERNEL);
6678 if (!doms)
6679 return NULL;
6680 for (i = 0; i < ndoms; i++) {
6681 if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) {
6682 free_sched_domains(doms, i);
6683 return NULL;
6684 }
6685 }
6686 return doms;
6687}
6688
6689void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms)
6690{
6691 unsigned int i;
6692 for (i = 0; i < ndoms; i++)
6693 free_cpumask_var(doms[i]);
6694 kfree(doms);
6695}
6696
1a20ff27 6697/*
41a2d6cf 6698 * Set up scheduler domains and groups. Callers must hold the hotplug lock.
029190c5
PJ
6699 * For now this just excludes isolated cpus, but could be used to
6700 * exclude other special cases in the future.
1a20ff27 6701 */
c4a8849a 6702static int init_sched_domains(const struct cpumask *cpu_map)
1a20ff27 6703{
7378547f
MM
6704 int err;
6705
22e52b07 6706 arch_update_cpu_topology();
029190c5 6707 ndoms_cur = 1;
acc3f5d7 6708 doms_cur = alloc_sched_domains(ndoms_cur);
029190c5 6709 if (!doms_cur)
acc3f5d7
RR
6710 doms_cur = &fallback_doms;
6711 cpumask_andnot(doms_cur[0], cpu_map, cpu_isolated_map);
dce840a0 6712 err = build_sched_domains(doms_cur[0], NULL);
6382bc90 6713 register_sched_domain_sysctl();
7378547f
MM
6714
6715 return err;
1a20ff27
DG
6716}
6717
1a20ff27
DG
6718/*
6719 * Detach sched domains from a group of cpus specified in cpu_map
6720 * These cpus will now be attached to the NULL domain
6721 */
96f874e2 6722static void detach_destroy_domains(const struct cpumask *cpu_map)
1a20ff27
DG
6723{
6724 int i;
6725
dce840a0 6726 rcu_read_lock();
abcd083a 6727 for_each_cpu(i, cpu_map)
57d885fe 6728 cpu_attach_domain(NULL, &def_root_domain, i);
dce840a0 6729 rcu_read_unlock();
1a20ff27
DG
6730}
6731
1d3504fc
HS
6732/* handle null as "default" */
6733static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
6734 struct sched_domain_attr *new, int idx_new)
6735{
6736 struct sched_domain_attr tmp;
6737
6738 /* fast path */
6739 if (!new && !cur)
6740 return 1;
6741
6742 tmp = SD_ATTR_INIT;
6743 return !memcmp(cur ? (cur + idx_cur) : &tmp,
6744 new ? (new + idx_new) : &tmp,
6745 sizeof(struct sched_domain_attr));
6746}
6747
029190c5
PJ
6748/*
6749 * Partition sched domains as specified by the 'ndoms_new'
41a2d6cf 6750 * cpumasks in the array doms_new[] of cpumasks. This compares
029190c5
PJ
6751 * doms_new[] to the current sched domain partitioning, doms_cur[].
6752 * It destroys each deleted domain and builds each new domain.
6753 *
acc3f5d7 6754 * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'.
41a2d6cf
IM
6755 * The masks don't intersect (don't overlap.) We should setup one
6756 * sched domain for each mask. CPUs not in any of the cpumasks will
6757 * not be load balanced. If the same cpumask appears both in the
029190c5
PJ
6758 * current 'doms_cur' domains and in the new 'doms_new', we can leave
6759 * it as it is.
6760 *
acc3f5d7
RR
6761 * The passed in 'doms_new' should be allocated using
6762 * alloc_sched_domains. This routine takes ownership of it and will
6763 * free_sched_domains it when done with it. If the caller failed the
6764 * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1,
6765 * and partition_sched_domains() will fallback to the single partition
6766 * 'fallback_doms', it also forces the domains to be rebuilt.
029190c5 6767 *
96f874e2 6768 * If doms_new == NULL it will be replaced with cpu_online_mask.
700018e0
LZ
6769 * ndoms_new == 0 is a special case for destroying existing domains,
6770 * and it will not create the default domain.
dfb512ec 6771 *
029190c5
PJ
6772 * Call with hotplug lock held
6773 */
acc3f5d7 6774void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
1d3504fc 6775 struct sched_domain_attr *dattr_new)
029190c5 6776{
dfb512ec 6777 int i, j, n;
d65bd5ec 6778 int new_topology;
029190c5 6779
712555ee 6780 mutex_lock(&sched_domains_mutex);
a1835615 6781
7378547f
MM
6782 /* always unregister in case we don't destroy any domains */
6783 unregister_sched_domain_sysctl();
6784
d65bd5ec
HC
6785 /* Let architecture update cpu core mappings. */
6786 new_topology = arch_update_cpu_topology();
6787
dfb512ec 6788 n = doms_new ? ndoms_new : 0;
029190c5
PJ
6789
6790 /* Destroy deleted domains */
6791 for (i = 0; i < ndoms_cur; i++) {
d65bd5ec 6792 for (j = 0; j < n && !new_topology; j++) {
acc3f5d7 6793 if (cpumask_equal(doms_cur[i], doms_new[j])
1d3504fc 6794 && dattrs_equal(dattr_cur, i, dattr_new, j))
029190c5
PJ
6795 goto match1;
6796 }
6797 /* no match - a current sched domain not in new doms_new[] */
acc3f5d7 6798 detach_destroy_domains(doms_cur[i]);
029190c5
PJ
6799match1:
6800 ;
6801 }
6802
c8d2d47a 6803 n = ndoms_cur;
e761b772 6804 if (doms_new == NULL) {
c8d2d47a 6805 n = 0;
acc3f5d7 6806 doms_new = &fallback_doms;
6ad4c188 6807 cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map);
faa2f98f 6808 WARN_ON_ONCE(dattr_new);
e761b772
MK
6809 }
6810
029190c5
PJ
6811 /* Build new domains */
6812 for (i = 0; i < ndoms_new; i++) {
c8d2d47a 6813 for (j = 0; j < n && !new_topology; j++) {
acc3f5d7 6814 if (cpumask_equal(doms_new[i], doms_cur[j])
1d3504fc 6815 && dattrs_equal(dattr_new, i, dattr_cur, j))
029190c5
PJ
6816 goto match2;
6817 }
6818 /* no match - add a new doms_new */
dce840a0 6819 build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL);
029190c5
PJ
6820match2:
6821 ;
6822 }
6823
6824 /* Remember the new sched domains */
acc3f5d7
RR
6825 if (doms_cur != &fallback_doms)
6826 free_sched_domains(doms_cur, ndoms_cur);
1d3504fc 6827 kfree(dattr_cur); /* kfree(NULL) is safe */
029190c5 6828 doms_cur = doms_new;
1d3504fc 6829 dattr_cur = dattr_new;
029190c5 6830 ndoms_cur = ndoms_new;
7378547f
MM
6831
6832 register_sched_domain_sysctl();
a1835615 6833
712555ee 6834 mutex_unlock(&sched_domains_mutex);
029190c5
PJ
6835}
6836
d35be8ba
SB
6837static int num_cpus_frozen; /* used to mark begin/end of suspend/resume */
6838
1da177e4 6839/*
3a101d05
TH
6840 * Update cpusets according to cpu_active mask. If cpusets are
6841 * disabled, cpuset_update_active_cpus() becomes a simple wrapper
6842 * around partition_sched_domains().
d35be8ba
SB
6843 *
6844 * If we come here as part of a suspend/resume, don't touch cpusets because we
6845 * want to restore it back to its original state upon resume anyway.
1da177e4 6846 */
0b2e918a
TH
6847static int cpuset_cpu_active(struct notifier_block *nfb, unsigned long action,
6848 void *hcpu)
e761b772 6849{
d35be8ba
SB
6850 switch (action) {
6851 case CPU_ONLINE_FROZEN:
6852 case CPU_DOWN_FAILED_FROZEN:
6853
6854 /*
6855 * num_cpus_frozen tracks how many CPUs are involved in suspend
6856 * resume sequence. As long as this is not the last online
6857 * operation in the resume sequence, just build a single sched
6858 * domain, ignoring cpusets.
6859 */
6860 num_cpus_frozen--;
6861 if (likely(num_cpus_frozen)) {
6862 partition_sched_domains(1, NULL, NULL);
6863 break;
6864 }
6865
6866 /*
6867 * This is the last CPU online operation. So fall through and
6868 * restore the original sched domains by considering the
6869 * cpuset configurations.
6870 */
6871
e761b772 6872 case CPU_ONLINE:
6ad4c188 6873 case CPU_DOWN_FAILED:
7ddf96b0 6874 cpuset_update_active_cpus(true);
d35be8ba 6875 break;
3a101d05
TH
6876 default:
6877 return NOTIFY_DONE;
6878 }
d35be8ba 6879 return NOTIFY_OK;
3a101d05 6880}
e761b772 6881
0b2e918a
TH
6882static int cpuset_cpu_inactive(struct notifier_block *nfb, unsigned long action,
6883 void *hcpu)
3a101d05 6884{
d35be8ba 6885 switch (action) {
3a101d05 6886 case CPU_DOWN_PREPARE:
7ddf96b0 6887 cpuset_update_active_cpus(false);
d35be8ba
SB
6888 break;
6889 case CPU_DOWN_PREPARE_FROZEN:
6890 num_cpus_frozen++;
6891 partition_sched_domains(1, NULL, NULL);
6892 break;
e761b772
MK
6893 default:
6894 return NOTIFY_DONE;
6895 }
d35be8ba 6896 return NOTIFY_OK;
e761b772 6897}
e761b772 6898
1da177e4
LT
6899void __init sched_init_smp(void)
6900{
dcc30a35
RR
6901 cpumask_var_t non_isolated_cpus;
6902
6903 alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
cb5fd13f 6904 alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
5c1e1767 6905
cb83b629
PZ
6906 sched_init_numa();
6907
6acce3ef
PZ
6908 /*
6909 * There's no userspace yet to cause hotplug operations; hence all the
6910 * cpu masks are stable and all blatant races in the below code cannot
6911 * happen.
6912 */
712555ee 6913 mutex_lock(&sched_domains_mutex);
c4a8849a 6914 init_sched_domains(cpu_active_mask);
dcc30a35
RR
6915 cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
6916 if (cpumask_empty(non_isolated_cpus))
6917 cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
712555ee 6918 mutex_unlock(&sched_domains_mutex);
e761b772 6919
301a5cba 6920 hotcpu_notifier(sched_domains_numa_masks_update, CPU_PRI_SCHED_ACTIVE);
3a101d05
TH
6921 hotcpu_notifier(cpuset_cpu_active, CPU_PRI_CPUSET_ACTIVE);
6922 hotcpu_notifier(cpuset_cpu_inactive, CPU_PRI_CPUSET_INACTIVE);
e761b772 6923
b328ca18 6924 init_hrtick();
5c1e1767
NP
6925
6926 /* Move init over to a non-isolated CPU */
dcc30a35 6927 if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
5c1e1767 6928 BUG();
19978ca6 6929 sched_init_granularity();
dcc30a35 6930 free_cpumask_var(non_isolated_cpus);
4212823f 6931
0e3900e6 6932 init_sched_rt_class();
1baca4ce 6933 init_sched_dl_class();
1da177e4
LT
6934}
6935#else
6936void __init sched_init_smp(void)
6937{
19978ca6 6938 sched_init_granularity();
1da177e4
LT
6939}
6940#endif /* CONFIG_SMP */
6941
cd1bb94b
AB
6942const_debug unsigned int sysctl_timer_migration = 1;
6943
1da177e4
LT
6944int in_sched_functions(unsigned long addr)
6945{
1da177e4
LT
6946 return in_lock_functions(addr) ||
6947 (addr >= (unsigned long)__sched_text_start
6948 && addr < (unsigned long)__sched_text_end);
6949}
6950
029632fb 6951#ifdef CONFIG_CGROUP_SCHED
27b4b931
LZ
6952/*
6953 * Default task group.
6954 * Every task in system belongs to this group at bootup.
6955 */
029632fb 6956struct task_group root_task_group;
35cf4e50 6957LIST_HEAD(task_groups);
052f1dc7 6958#endif
6f505b16 6959
e6252c3e 6960DECLARE_PER_CPU(cpumask_var_t, load_balance_mask);
6f505b16 6961
1da177e4
LT
6962void __init sched_init(void)
6963{
dd41f596 6964 int i, j;
434d53b0
MT
6965 unsigned long alloc_size = 0, ptr;
6966
6967#ifdef CONFIG_FAIR_GROUP_SCHED
6968 alloc_size += 2 * nr_cpu_ids * sizeof(void **);
6969#endif
6970#ifdef CONFIG_RT_GROUP_SCHED
6971 alloc_size += 2 * nr_cpu_ids * sizeof(void **);
eff766a6 6972#endif
df7c8e84 6973#ifdef CONFIG_CPUMASK_OFFSTACK
8c083f08 6974 alloc_size += num_possible_cpus() * cpumask_size();
434d53b0 6975#endif
434d53b0 6976 if (alloc_size) {
36b7b6d4 6977 ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT);
434d53b0
MT
6978
6979#ifdef CONFIG_FAIR_GROUP_SCHED
07e06b01 6980 root_task_group.se = (struct sched_entity **)ptr;
434d53b0
MT
6981 ptr += nr_cpu_ids * sizeof(void **);
6982
07e06b01 6983 root_task_group.cfs_rq = (struct cfs_rq **)ptr;
434d53b0 6984 ptr += nr_cpu_ids * sizeof(void **);
eff766a6 6985
6d6bc0ad 6986#endif /* CONFIG_FAIR_GROUP_SCHED */
434d53b0 6987#ifdef CONFIG_RT_GROUP_SCHED
07e06b01 6988 root_task_group.rt_se = (struct sched_rt_entity **)ptr;
434d53b0
MT
6989 ptr += nr_cpu_ids * sizeof(void **);
6990
07e06b01 6991 root_task_group.rt_rq = (struct rt_rq **)ptr;
eff766a6
PZ
6992 ptr += nr_cpu_ids * sizeof(void **);
6993
6d6bc0ad 6994#endif /* CONFIG_RT_GROUP_SCHED */
df7c8e84
RR
6995#ifdef CONFIG_CPUMASK_OFFSTACK
6996 for_each_possible_cpu(i) {
e6252c3e 6997 per_cpu(load_balance_mask, i) = (void *)ptr;
df7c8e84
RR
6998 ptr += cpumask_size();
6999 }
7000#endif /* CONFIG_CPUMASK_OFFSTACK */
434d53b0 7001 }
dd41f596 7002
332ac17e
DF
7003 init_rt_bandwidth(&def_rt_bandwidth,
7004 global_rt_period(), global_rt_runtime());
7005 init_dl_bandwidth(&def_dl_bandwidth,
1724813d 7006 global_rt_period(), global_rt_runtime());
332ac17e 7007
57d885fe
GH
7008#ifdef CONFIG_SMP
7009 init_defrootdomain();
7010#endif
7011
d0b27fa7 7012#ifdef CONFIG_RT_GROUP_SCHED
07e06b01 7013 init_rt_bandwidth(&root_task_group.rt_bandwidth,
d0b27fa7 7014 global_rt_period(), global_rt_runtime());
6d6bc0ad 7015#endif /* CONFIG_RT_GROUP_SCHED */
d0b27fa7 7016
7c941438 7017#ifdef CONFIG_CGROUP_SCHED
07e06b01
YZ
7018 list_add(&root_task_group.list, &task_groups);
7019 INIT_LIST_HEAD(&root_task_group.children);
f4d6f6c2 7020 INIT_LIST_HEAD(&root_task_group.siblings);
5091faa4 7021 autogroup_init(&init_task);
54c707e9 7022
7c941438 7023#endif /* CONFIG_CGROUP_SCHED */
6f505b16 7024
0a945022 7025 for_each_possible_cpu(i) {
70b97a7f 7026 struct rq *rq;
1da177e4
LT
7027
7028 rq = cpu_rq(i);
05fa785c 7029 raw_spin_lock_init(&rq->lock);
7897986b 7030 rq->nr_running = 0;
dce48a84
TG
7031 rq->calc_load_active = 0;
7032 rq->calc_load_update = jiffies + LOAD_FREQ;
acb5a9ba 7033 init_cfs_rq(&rq->cfs);
6f505b16 7034 init_rt_rq(&rq->rt, rq);
aab03e05 7035 init_dl_rq(&rq->dl, rq);
dd41f596 7036#ifdef CONFIG_FAIR_GROUP_SCHED
029632fb 7037 root_task_group.shares = ROOT_TASK_GROUP_LOAD;
6f505b16 7038 INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
354d60c2 7039 /*
07e06b01 7040 * How much cpu bandwidth does root_task_group get?
354d60c2
DG
7041 *
7042 * In case of task-groups formed thr' the cgroup filesystem, it
7043 * gets 100% of the cpu resources in the system. This overall
7044 * system cpu resource is divided among the tasks of
07e06b01 7045 * root_task_group and its child task-groups in a fair manner,
354d60c2
DG
7046 * based on each entity's (task or task-group's) weight
7047 * (se->load.weight).
7048 *
07e06b01 7049 * In other words, if root_task_group has 10 tasks of weight
354d60c2
DG
7050 * 1024) and two child groups A0 and A1 (of weight 1024 each),
7051 * then A0's share of the cpu resource is:
7052 *
0d905bca 7053 * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
354d60c2 7054 *
07e06b01
YZ
7055 * We achieve this by letting root_task_group's tasks sit
7056 * directly in rq->cfs (i.e root_task_group->se[] = NULL).
354d60c2 7057 */
ab84d31e 7058 init_cfs_bandwidth(&root_task_group.cfs_bandwidth);
07e06b01 7059 init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL);
354d60c2
DG
7060#endif /* CONFIG_FAIR_GROUP_SCHED */
7061
7062 rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
052f1dc7 7063#ifdef CONFIG_RT_GROUP_SCHED
07e06b01 7064 init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL);
dd41f596 7065#endif
1da177e4 7066
dd41f596
IM
7067 for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
7068 rq->cpu_load[j] = 0;
fdf3e95d
VP
7069
7070 rq->last_load_update_tick = jiffies;
7071
1da177e4 7072#ifdef CONFIG_SMP
41c7ce9a 7073 rq->sd = NULL;
57d885fe 7074 rq->rd = NULL;
ca8ce3d0 7075 rq->cpu_capacity = SCHED_CAPACITY_SCALE;
3f029d3c 7076 rq->post_schedule = 0;
1da177e4 7077 rq->active_balance = 0;
dd41f596 7078 rq->next_balance = jiffies;
1da177e4 7079 rq->push_cpu = 0;
0a2966b4 7080 rq->cpu = i;
1f11eb6a 7081 rq->online = 0;
eae0c9df
MG
7082 rq->idle_stamp = 0;
7083 rq->avg_idle = 2*sysctl_sched_migration_cost;
9bd721c5 7084 rq->max_idle_balance_cost = sysctl_sched_migration_cost;
367456c7
PZ
7085
7086 INIT_LIST_HEAD(&rq->cfs_tasks);
7087
dc938520 7088 rq_attach_root(rq, &def_root_domain);
3451d024 7089#ifdef CONFIG_NO_HZ_COMMON
1c792db7 7090 rq->nohz_flags = 0;
83cd4fe2 7091#endif
265f22a9
FW
7092#ifdef CONFIG_NO_HZ_FULL
7093 rq->last_sched_tick = 0;
7094#endif
1da177e4 7095#endif
8f4d37ec 7096 init_rq_hrtick(rq);
1da177e4 7097 atomic_set(&rq->nr_iowait, 0);
1da177e4
LT
7098 }
7099
2dd73a4f 7100 set_load_weight(&init_task);
b50f60ce 7101
e107be36
AK
7102#ifdef CONFIG_PREEMPT_NOTIFIERS
7103 INIT_HLIST_HEAD(&init_task.preempt_notifiers);
7104#endif
7105
1da177e4
LT
7106 /*
7107 * The boot idle thread does lazy MMU switching as well:
7108 */
7109 atomic_inc(&init_mm.mm_count);
7110 enter_lazy_tlb(&init_mm, current);
7111
7112 /*
7113 * Make us the idle thread. Technically, schedule() should not be
7114 * called from this thread, however somewhere below it might be,
7115 * but because we are the idle thread, we just pick up running again
7116 * when this runqueue becomes "idle".
7117 */
7118 init_idle(current, smp_processor_id());
dce48a84
TG
7119
7120 calc_load_update = jiffies + LOAD_FREQ;
7121
dd41f596
IM
7122 /*
7123 * During early bootup we pretend to be a normal task:
7124 */
7125 current->sched_class = &fair_sched_class;
6892b75e 7126
bf4d83f6 7127#ifdef CONFIG_SMP
4cb98839 7128 zalloc_cpumask_var(&sched_domains_tmpmask, GFP_NOWAIT);
bdddd296
RR
7129 /* May be allocated at isolcpus cmdline parse time */
7130 if (cpu_isolated_map == NULL)
7131 zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT);
29d5e047 7132 idle_thread_set_boot_cpu();
a803f026 7133 set_cpu_rq_start_time();
029632fb
PZ
7134#endif
7135 init_sched_fair_class();
6a7b3dc3 7136
6892b75e 7137 scheduler_running = 1;
1da177e4
LT
7138}
7139
d902db1e 7140#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
e4aafea2
FW
7141static inline int preempt_count_equals(int preempt_offset)
7142{
234da7bc 7143 int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth();
e4aafea2 7144
4ba8216c 7145 return (nested == preempt_offset);
e4aafea2
FW
7146}
7147
d894837f 7148void __might_sleep(const char *file, int line, int preempt_offset)
1da177e4 7149{
1da177e4
LT
7150 static unsigned long prev_jiffy; /* ratelimiting */
7151
b3fbab05 7152 rcu_sleep_check(); /* WARN_ON_ONCE() by default, no rate limit reqd. */
db273be2
TG
7153 if ((preempt_count_equals(preempt_offset) && !irqs_disabled() &&
7154 !is_idle_task(current)) ||
e4aafea2 7155 system_state != SYSTEM_RUNNING || oops_in_progress)
aef745fc
IM
7156 return;
7157 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
7158 return;
7159 prev_jiffy = jiffies;
7160
3df0fc5b
PZ
7161 printk(KERN_ERR
7162 "BUG: sleeping function called from invalid context at %s:%d\n",
7163 file, line);
7164 printk(KERN_ERR
7165 "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
7166 in_atomic(), irqs_disabled(),
7167 current->pid, current->comm);
aef745fc
IM
7168
7169 debug_show_held_locks(current);
7170 if (irqs_disabled())
7171 print_irqtrace_events(current);
8f47b187
TG
7172#ifdef CONFIG_DEBUG_PREEMPT
7173 if (!preempt_count_equals(preempt_offset)) {
7174 pr_err("Preemption disabled at:");
7175 print_ip_sym(current->preempt_disable_ip);
7176 pr_cont("\n");
7177 }
7178#endif
aef745fc 7179 dump_stack();
1da177e4
LT
7180}
7181EXPORT_SYMBOL(__might_sleep);
7182#endif
7183
7184#ifdef CONFIG_MAGIC_SYSRQ
3a5e4dc1
AK
7185static void normalize_task(struct rq *rq, struct task_struct *p)
7186{
da7a735e 7187 const struct sched_class *prev_class = p->sched_class;
d50dde5a
DF
7188 struct sched_attr attr = {
7189 .sched_policy = SCHED_NORMAL,
7190 };
da7a735e 7191 int old_prio = p->prio;
da0c1e65 7192 int queued;
3e51f33f 7193
da0c1e65
KT
7194 queued = task_on_rq_queued(p);
7195 if (queued)
4ca9b72b 7196 dequeue_task(rq, p, 0);
d50dde5a 7197 __setscheduler(rq, p, &attr);
da0c1e65 7198 if (queued) {
4ca9b72b 7199 enqueue_task(rq, p, 0);
8875125e 7200 resched_curr(rq);
3a5e4dc1 7201 }
da7a735e
PZ
7202
7203 check_class_changed(rq, p, prev_class, old_prio);
3a5e4dc1
AK
7204}
7205
1da177e4
LT
7206void normalize_rt_tasks(void)
7207{
a0f98a1c 7208 struct task_struct *g, *p;
1da177e4 7209 unsigned long flags;
70b97a7f 7210 struct rq *rq;
1da177e4 7211
4cf5d77a 7212 read_lock_irqsave(&tasklist_lock, flags);
5d07f420 7213 for_each_process_thread(g, p) {
178be793
IM
7214 /*
7215 * Only normalize user tasks:
7216 */
7217 if (!p->mm)
7218 continue;
7219
6cfb0d5d 7220 p->se.exec_start = 0;
6cfb0d5d 7221#ifdef CONFIG_SCHEDSTATS
41acab88
LDM
7222 p->se.statistics.wait_start = 0;
7223 p->se.statistics.sleep_start = 0;
7224 p->se.statistics.block_start = 0;
6cfb0d5d 7225#endif
dd41f596 7226
aab03e05 7227 if (!dl_task(p) && !rt_task(p)) {
dd41f596
IM
7228 /*
7229 * Renice negative nice level userspace
7230 * tasks back to 0:
7231 */
d0ea0268 7232 if (task_nice(p) < 0 && p->mm)
dd41f596 7233 set_user_nice(p, 0);
1da177e4 7234 continue;
dd41f596 7235 }
1da177e4 7236
1d615482 7237 raw_spin_lock(&p->pi_lock);
b29739f9 7238 rq = __task_rq_lock(p);
1da177e4 7239
178be793 7240 normalize_task(rq, p);
3a5e4dc1 7241
b29739f9 7242 __task_rq_unlock(rq);
1d615482 7243 raw_spin_unlock(&p->pi_lock);
5d07f420 7244 }
4cf5d77a 7245 read_unlock_irqrestore(&tasklist_lock, flags);
1da177e4
LT
7246}
7247
7248#endif /* CONFIG_MAGIC_SYSRQ */
1df5c10a 7249
67fc4e0c 7250#if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB)
1df5c10a 7251/*
67fc4e0c 7252 * These functions are only useful for the IA64 MCA handling, or kdb.
1df5c10a
LT
7253 *
7254 * They can only be called when the whole system has been
7255 * stopped - every CPU needs to be quiescent, and no scheduling
7256 * activity can take place. Using them for anything else would
7257 * be a serious bug, and as a result, they aren't even visible
7258 * under any other configuration.
7259 */
7260
7261/**
7262 * curr_task - return the current task for a given cpu.
7263 * @cpu: the processor in question.
7264 *
7265 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
e69f6186
YB
7266 *
7267 * Return: The current task for @cpu.
1df5c10a 7268 */
36c8b586 7269struct task_struct *curr_task(int cpu)
1df5c10a
LT
7270{
7271 return cpu_curr(cpu);
7272}
7273
67fc4e0c
JW
7274#endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */
7275
7276#ifdef CONFIG_IA64
1df5c10a
LT
7277/**
7278 * set_curr_task - set the current task for a given cpu.
7279 * @cpu: the processor in question.
7280 * @p: the task pointer to set.
7281 *
7282 * Description: This function must only be used when non-maskable interrupts
41a2d6cf
IM
7283 * are serviced on a separate stack. It allows the architecture to switch the
7284 * notion of the current task on a cpu in a non-blocking manner. This function
1df5c10a
LT
7285 * must be called with all CPU's synchronized, and interrupts disabled, the
7286 * and caller must save the original value of the current task (see
7287 * curr_task() above) and restore that value before reenabling interrupts and
7288 * re-starting the system.
7289 *
7290 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
7291 */
36c8b586 7292void set_curr_task(int cpu, struct task_struct *p)
1df5c10a
LT
7293{
7294 cpu_curr(cpu) = p;
7295}
7296
7297#endif
29f59db3 7298
7c941438 7299#ifdef CONFIG_CGROUP_SCHED
029632fb
PZ
7300/* task_group_lock serializes the addition/removal of task groups */
7301static DEFINE_SPINLOCK(task_group_lock);
7302
bccbe08a
PZ
7303static void free_sched_group(struct task_group *tg)
7304{
7305 free_fair_sched_group(tg);
7306 free_rt_sched_group(tg);
e9aa1dd1 7307 autogroup_free(tg);
bccbe08a
PZ
7308 kfree(tg);
7309}
7310
7311/* allocate runqueue etc for a new task group */
ec7dc8ac 7312struct task_group *sched_create_group(struct task_group *parent)
bccbe08a
PZ
7313{
7314 struct task_group *tg;
bccbe08a
PZ
7315
7316 tg = kzalloc(sizeof(*tg), GFP_KERNEL);
7317 if (!tg)
7318 return ERR_PTR(-ENOMEM);
7319
ec7dc8ac 7320 if (!alloc_fair_sched_group(tg, parent))
bccbe08a
PZ
7321 goto err;
7322
ec7dc8ac 7323 if (!alloc_rt_sched_group(tg, parent))
bccbe08a
PZ
7324 goto err;
7325
ace783b9
LZ
7326 return tg;
7327
7328err:
7329 free_sched_group(tg);
7330 return ERR_PTR(-ENOMEM);
7331}
7332
7333void sched_online_group(struct task_group *tg, struct task_group *parent)
7334{
7335 unsigned long flags;
7336
8ed36996 7337 spin_lock_irqsave(&task_group_lock, flags);
6f505b16 7338 list_add_rcu(&tg->list, &task_groups);
f473aa5e
PZ
7339
7340 WARN_ON(!parent); /* root should already exist */
7341
7342 tg->parent = parent;
f473aa5e 7343 INIT_LIST_HEAD(&tg->children);
09f2724a 7344 list_add_rcu(&tg->siblings, &parent->children);
8ed36996 7345 spin_unlock_irqrestore(&task_group_lock, flags);
29f59db3
SV
7346}
7347
9b5b7751 7348/* rcu callback to free various structures associated with a task group */
6f505b16 7349static void free_sched_group_rcu(struct rcu_head *rhp)
29f59db3 7350{
29f59db3 7351 /* now it should be safe to free those cfs_rqs */
6f505b16 7352 free_sched_group(container_of(rhp, struct task_group, rcu));
29f59db3
SV
7353}
7354
9b5b7751 7355/* Destroy runqueue etc associated with a task group */
4cf86d77 7356void sched_destroy_group(struct task_group *tg)
ace783b9
LZ
7357{
7358 /* wait for possible concurrent references to cfs_rqs complete */
7359 call_rcu(&tg->rcu, free_sched_group_rcu);
7360}
7361
7362void sched_offline_group(struct task_group *tg)
29f59db3 7363{
8ed36996 7364 unsigned long flags;
9b5b7751 7365 int i;
29f59db3 7366
3d4b47b4
PZ
7367 /* end participation in shares distribution */
7368 for_each_possible_cpu(i)
bccbe08a 7369 unregister_fair_sched_group(tg, i);
3d4b47b4
PZ
7370
7371 spin_lock_irqsave(&task_group_lock, flags);
6f505b16 7372 list_del_rcu(&tg->list);
f473aa5e 7373 list_del_rcu(&tg->siblings);
8ed36996 7374 spin_unlock_irqrestore(&task_group_lock, flags);
29f59db3
SV
7375}
7376
9b5b7751 7377/* change task's runqueue when it moves between groups.
3a252015
IM
7378 * The caller of this function should have put the task in its new group
7379 * by now. This function just updates tsk->se.cfs_rq and tsk->se.parent to
7380 * reflect its new group.
9b5b7751
SV
7381 */
7382void sched_move_task(struct task_struct *tsk)
29f59db3 7383{
8323f26c 7384 struct task_group *tg;
da0c1e65 7385 int queued, running;
29f59db3
SV
7386 unsigned long flags;
7387 struct rq *rq;
7388
7389 rq = task_rq_lock(tsk, &flags);
7390
051a1d1a 7391 running = task_current(rq, tsk);
da0c1e65 7392 queued = task_on_rq_queued(tsk);
29f59db3 7393
da0c1e65 7394 if (queued)
29f59db3 7395 dequeue_task(rq, tsk, 0);
0e1f3483 7396 if (unlikely(running))
f3cd1c4e 7397 put_prev_task(rq, tsk);
29f59db3 7398
073219e9 7399 tg = container_of(task_css_check(tsk, cpu_cgrp_id,
8323f26c
PZ
7400 lockdep_is_held(&tsk->sighand->siglock)),
7401 struct task_group, css);
7402 tg = autogroup_task_group(tsk, tg);
7403 tsk->sched_task_group = tg;
7404
810b3817 7405#ifdef CONFIG_FAIR_GROUP_SCHED
b2b5ce02 7406 if (tsk->sched_class->task_move_group)
da0c1e65 7407 tsk->sched_class->task_move_group(tsk, queued);
b2b5ce02 7408 else
810b3817 7409#endif
b2b5ce02 7410 set_task_rq(tsk, task_cpu(tsk));
810b3817 7411
0e1f3483
HS
7412 if (unlikely(running))
7413 tsk->sched_class->set_curr_task(rq);
da0c1e65 7414 if (queued)
371fd7e7 7415 enqueue_task(rq, tsk, 0);
29f59db3 7416
0122ec5b 7417 task_rq_unlock(rq, tsk, &flags);
29f59db3 7418}
7c941438 7419#endif /* CONFIG_CGROUP_SCHED */
29f59db3 7420
a790de99
PT
7421#ifdef CONFIG_RT_GROUP_SCHED
7422/*
7423 * Ensure that the real time constraints are schedulable.
7424 */
7425static DEFINE_MUTEX(rt_constraints_mutex);
9f0c1e56 7426
9a7e0b18
PZ
7427/* Must be called with tasklist_lock held */
7428static inline int tg_has_rt_tasks(struct task_group *tg)
b40b2e8e 7429{
9a7e0b18 7430 struct task_struct *g, *p;
b40b2e8e 7431
5d07f420 7432 for_each_process_thread(g, p) {
029632fb 7433 if (rt_task(p) && task_rq(p)->rt.tg == tg)
9a7e0b18 7434 return 1;
5d07f420 7435 }
b40b2e8e 7436
9a7e0b18
PZ
7437 return 0;
7438}
b40b2e8e 7439
9a7e0b18
PZ
7440struct rt_schedulable_data {
7441 struct task_group *tg;
7442 u64 rt_period;
7443 u64 rt_runtime;
7444};
b40b2e8e 7445
a790de99 7446static int tg_rt_schedulable(struct task_group *tg, void *data)
9a7e0b18
PZ
7447{
7448 struct rt_schedulable_data *d = data;
7449 struct task_group *child;
7450 unsigned long total, sum = 0;
7451 u64 period, runtime;
b40b2e8e 7452
9a7e0b18
PZ
7453 period = ktime_to_ns(tg->rt_bandwidth.rt_period);
7454 runtime = tg->rt_bandwidth.rt_runtime;
b40b2e8e 7455
9a7e0b18
PZ
7456 if (tg == d->tg) {
7457 period = d->rt_period;
7458 runtime = d->rt_runtime;
b40b2e8e 7459 }
b40b2e8e 7460
4653f803
PZ
7461 /*
7462 * Cannot have more runtime than the period.
7463 */
7464 if (runtime > period && runtime != RUNTIME_INF)
7465 return -EINVAL;
6f505b16 7466
4653f803
PZ
7467 /*
7468 * Ensure we don't starve existing RT tasks.
7469 */
9a7e0b18
PZ
7470 if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg))
7471 return -EBUSY;
6f505b16 7472
9a7e0b18 7473 total = to_ratio(period, runtime);
6f505b16 7474
4653f803
PZ
7475 /*
7476 * Nobody can have more than the global setting allows.
7477 */
7478 if (total > to_ratio(global_rt_period(), global_rt_runtime()))
7479 return -EINVAL;
6f505b16 7480
4653f803
PZ
7481 /*
7482 * The sum of our children's runtime should not exceed our own.
7483 */
9a7e0b18
PZ
7484 list_for_each_entry_rcu(child, &tg->children, siblings) {
7485 period = ktime_to_ns(child->rt_bandwidth.rt_period);
7486 runtime = child->rt_bandwidth.rt_runtime;
6f505b16 7487
9a7e0b18
PZ
7488 if (child == d->tg) {
7489 period = d->rt_period;
7490 runtime = d->rt_runtime;
7491 }
6f505b16 7492
9a7e0b18 7493 sum += to_ratio(period, runtime);
9f0c1e56 7494 }
6f505b16 7495
9a7e0b18
PZ
7496 if (sum > total)
7497 return -EINVAL;
7498
7499 return 0;
6f505b16
PZ
7500}
7501
9a7e0b18 7502static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
521f1a24 7503{
8277434e
PT
7504 int ret;
7505
9a7e0b18
PZ
7506 struct rt_schedulable_data data = {
7507 .tg = tg,
7508 .rt_period = period,
7509 .rt_runtime = runtime,
7510 };
7511
8277434e
PT
7512 rcu_read_lock();
7513 ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data);
7514 rcu_read_unlock();
7515
7516 return ret;
521f1a24
DG
7517}
7518
ab84d31e 7519static int tg_set_rt_bandwidth(struct task_group *tg,
d0b27fa7 7520 u64 rt_period, u64 rt_runtime)
6f505b16 7521{
ac086bc2 7522 int i, err = 0;
9f0c1e56 7523
9f0c1e56 7524 mutex_lock(&rt_constraints_mutex);
521f1a24 7525 read_lock(&tasklist_lock);
9a7e0b18
PZ
7526 err = __rt_schedulable(tg, rt_period, rt_runtime);
7527 if (err)
9f0c1e56 7528 goto unlock;
ac086bc2 7529
0986b11b 7530 raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
d0b27fa7
PZ
7531 tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
7532 tg->rt_bandwidth.rt_runtime = rt_runtime;
ac086bc2
PZ
7533
7534 for_each_possible_cpu(i) {
7535 struct rt_rq *rt_rq = tg->rt_rq[i];
7536
0986b11b 7537 raw_spin_lock(&rt_rq->rt_runtime_lock);
ac086bc2 7538 rt_rq->rt_runtime = rt_runtime;
0986b11b 7539 raw_spin_unlock(&rt_rq->rt_runtime_lock);
ac086bc2 7540 }
0986b11b 7541 raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
49246274 7542unlock:
521f1a24 7543 read_unlock(&tasklist_lock);
9f0c1e56
PZ
7544 mutex_unlock(&rt_constraints_mutex);
7545
7546 return err;
6f505b16
PZ
7547}
7548
25cc7da7 7549static int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
d0b27fa7
PZ
7550{
7551 u64 rt_runtime, rt_period;
7552
7553 rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
7554 rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
7555 if (rt_runtime_us < 0)
7556 rt_runtime = RUNTIME_INF;
7557
ab84d31e 7558 return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
d0b27fa7
PZ
7559}
7560
25cc7da7 7561static long sched_group_rt_runtime(struct task_group *tg)
9f0c1e56
PZ
7562{
7563 u64 rt_runtime_us;
7564
d0b27fa7 7565 if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
9f0c1e56
PZ
7566 return -1;
7567
d0b27fa7 7568 rt_runtime_us = tg->rt_bandwidth.rt_runtime;
9f0c1e56
PZ
7569 do_div(rt_runtime_us, NSEC_PER_USEC);
7570 return rt_runtime_us;
7571}
d0b27fa7 7572
25cc7da7 7573static int sched_group_set_rt_period(struct task_group *tg, long rt_period_us)
d0b27fa7
PZ
7574{
7575 u64 rt_runtime, rt_period;
7576
7577 rt_period = (u64)rt_period_us * NSEC_PER_USEC;
7578 rt_runtime = tg->rt_bandwidth.rt_runtime;
7579
619b0488
R
7580 if (rt_period == 0)
7581 return -EINVAL;
7582
ab84d31e 7583 return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
d0b27fa7
PZ
7584}
7585
25cc7da7 7586static long sched_group_rt_period(struct task_group *tg)
d0b27fa7
PZ
7587{
7588 u64 rt_period_us;
7589
7590 rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
7591 do_div(rt_period_us, NSEC_PER_USEC);
7592 return rt_period_us;
7593}
332ac17e 7594#endif /* CONFIG_RT_GROUP_SCHED */
d0b27fa7 7595
332ac17e 7596#ifdef CONFIG_RT_GROUP_SCHED
d0b27fa7
PZ
7597static int sched_rt_global_constraints(void)
7598{
7599 int ret = 0;
7600
7601 mutex_lock(&rt_constraints_mutex);
9a7e0b18 7602 read_lock(&tasklist_lock);
4653f803 7603 ret = __rt_schedulable(NULL, 0, 0);
9a7e0b18 7604 read_unlock(&tasklist_lock);
d0b27fa7
PZ
7605 mutex_unlock(&rt_constraints_mutex);
7606
7607 return ret;
7608}
54e99124 7609
25cc7da7 7610static int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk)
54e99124
DG
7611{
7612 /* Don't accept realtime tasks when there is no way for them to run */
7613 if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0)
7614 return 0;
7615
7616 return 1;
7617}
7618
6d6bc0ad 7619#else /* !CONFIG_RT_GROUP_SCHED */
d0b27fa7
PZ
7620static int sched_rt_global_constraints(void)
7621{
ac086bc2 7622 unsigned long flags;
332ac17e 7623 int i, ret = 0;
ec5d4989 7624
0986b11b 7625 raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
ac086bc2
PZ
7626 for_each_possible_cpu(i) {
7627 struct rt_rq *rt_rq = &cpu_rq(i)->rt;
7628
0986b11b 7629 raw_spin_lock(&rt_rq->rt_runtime_lock);
ac086bc2 7630 rt_rq->rt_runtime = global_rt_runtime();
0986b11b 7631 raw_spin_unlock(&rt_rq->rt_runtime_lock);
ac086bc2 7632 }
0986b11b 7633 raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
ac086bc2 7634
332ac17e 7635 return ret;
d0b27fa7 7636}
6d6bc0ad 7637#endif /* CONFIG_RT_GROUP_SCHED */
d0b27fa7 7638
332ac17e
DF
7639static int sched_dl_global_constraints(void)
7640{
1724813d
PZ
7641 u64 runtime = global_rt_runtime();
7642 u64 period = global_rt_period();
332ac17e 7643 u64 new_bw = to_ratio(period, runtime);
1724813d 7644 int cpu, ret = 0;
49516342 7645 unsigned long flags;
332ac17e
DF
7646
7647 /*
7648 * Here we want to check the bandwidth not being set to some
7649 * value smaller than the currently allocated bandwidth in
7650 * any of the root_domains.
7651 *
7652 * FIXME: Cycling on all the CPUs is overdoing, but simpler than
7653 * cycling on root_domains... Discussion on different/better
7654 * solutions is welcome!
7655 */
1724813d
PZ
7656 for_each_possible_cpu(cpu) {
7657 struct dl_bw *dl_b = dl_bw_of(cpu);
332ac17e 7658
49516342 7659 raw_spin_lock_irqsave(&dl_b->lock, flags);
1724813d
PZ
7660 if (new_bw < dl_b->total_bw)
7661 ret = -EBUSY;
49516342 7662 raw_spin_unlock_irqrestore(&dl_b->lock, flags);
1724813d
PZ
7663
7664 if (ret)
7665 break;
332ac17e
DF
7666 }
7667
1724813d 7668 return ret;
332ac17e
DF
7669}
7670
1724813d 7671static void sched_dl_do_global(void)
ce0dbbbb 7672{
1724813d
PZ
7673 u64 new_bw = -1;
7674 int cpu;
49516342 7675 unsigned long flags;
ce0dbbbb 7676
1724813d
PZ
7677 def_dl_bandwidth.dl_period = global_rt_period();
7678 def_dl_bandwidth.dl_runtime = global_rt_runtime();
7679
7680 if (global_rt_runtime() != RUNTIME_INF)
7681 new_bw = to_ratio(global_rt_period(), global_rt_runtime());
7682
7683 /*
7684 * FIXME: As above...
7685 */
7686 for_each_possible_cpu(cpu) {
7687 struct dl_bw *dl_b = dl_bw_of(cpu);
7688
49516342 7689 raw_spin_lock_irqsave(&dl_b->lock, flags);
1724813d 7690 dl_b->bw = new_bw;
49516342 7691 raw_spin_unlock_irqrestore(&dl_b->lock, flags);
ce0dbbbb 7692 }
1724813d
PZ
7693}
7694
7695static int sched_rt_global_validate(void)
7696{
7697 if (sysctl_sched_rt_period <= 0)
7698 return -EINVAL;
7699
e9e7cb38
JL
7700 if ((sysctl_sched_rt_runtime != RUNTIME_INF) &&
7701 (sysctl_sched_rt_runtime > sysctl_sched_rt_period))
1724813d
PZ
7702 return -EINVAL;
7703
7704 return 0;
7705}
7706
7707static void sched_rt_do_global(void)
7708{
7709 def_rt_bandwidth.rt_runtime = global_rt_runtime();
7710 def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period());
ce0dbbbb
CW
7711}
7712
d0b27fa7 7713int sched_rt_handler(struct ctl_table *table, int write,
8d65af78 7714 void __user *buffer, size_t *lenp,
d0b27fa7
PZ
7715 loff_t *ppos)
7716{
d0b27fa7
PZ
7717 int old_period, old_runtime;
7718 static DEFINE_MUTEX(mutex);
1724813d 7719 int ret;
d0b27fa7
PZ
7720
7721 mutex_lock(&mutex);
7722 old_period = sysctl_sched_rt_period;
7723 old_runtime = sysctl_sched_rt_runtime;
7724
8d65af78 7725 ret = proc_dointvec(table, write, buffer, lenp, ppos);
d0b27fa7
PZ
7726
7727 if (!ret && write) {
1724813d
PZ
7728 ret = sched_rt_global_validate();
7729 if (ret)
7730 goto undo;
7731
d0b27fa7 7732 ret = sched_rt_global_constraints();
1724813d
PZ
7733 if (ret)
7734 goto undo;
7735
7736 ret = sched_dl_global_constraints();
7737 if (ret)
7738 goto undo;
7739
7740 sched_rt_do_global();
7741 sched_dl_do_global();
7742 }
7743 if (0) {
7744undo:
7745 sysctl_sched_rt_period = old_period;
7746 sysctl_sched_rt_runtime = old_runtime;
d0b27fa7
PZ
7747 }
7748 mutex_unlock(&mutex);
7749
7750 return ret;
7751}
68318b8e 7752
1724813d 7753int sched_rr_handler(struct ctl_table *table, int write,
332ac17e
DF
7754 void __user *buffer, size_t *lenp,
7755 loff_t *ppos)
7756{
7757 int ret;
332ac17e 7758 static DEFINE_MUTEX(mutex);
332ac17e
DF
7759
7760 mutex_lock(&mutex);
332ac17e 7761 ret = proc_dointvec(table, write, buffer, lenp, ppos);
1724813d
PZ
7762 /* make sure that internally we keep jiffies */
7763 /* also, writing zero resets timeslice to default */
332ac17e 7764 if (!ret && write) {
1724813d
PZ
7765 sched_rr_timeslice = sched_rr_timeslice <= 0 ?
7766 RR_TIMESLICE : msecs_to_jiffies(sched_rr_timeslice);
332ac17e
DF
7767 }
7768 mutex_unlock(&mutex);
332ac17e
DF
7769 return ret;
7770}
7771
052f1dc7 7772#ifdef CONFIG_CGROUP_SCHED
68318b8e 7773
a7c6d554 7774static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
68318b8e 7775{
a7c6d554 7776 return css ? container_of(css, struct task_group, css) : NULL;
68318b8e
SV
7777}
7778
eb95419b
TH
7779static struct cgroup_subsys_state *
7780cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
68318b8e 7781{
eb95419b
TH
7782 struct task_group *parent = css_tg(parent_css);
7783 struct task_group *tg;
68318b8e 7784
eb95419b 7785 if (!parent) {
68318b8e 7786 /* This is early initialization for the top cgroup */
07e06b01 7787 return &root_task_group.css;
68318b8e
SV
7788 }
7789
ec7dc8ac 7790 tg = sched_create_group(parent);
68318b8e
SV
7791 if (IS_ERR(tg))
7792 return ERR_PTR(-ENOMEM);
7793
68318b8e
SV
7794 return &tg->css;
7795}
7796
eb95419b 7797static int cpu_cgroup_css_online(struct cgroup_subsys_state *css)
ace783b9 7798{
eb95419b 7799 struct task_group *tg = css_tg(css);
5c9d535b 7800 struct task_group *parent = css_tg(css->parent);
ace783b9 7801
63876986
TH
7802 if (parent)
7803 sched_online_group(tg, parent);
ace783b9
LZ
7804 return 0;
7805}
7806
eb95419b 7807static void cpu_cgroup_css_free(struct cgroup_subsys_state *css)
68318b8e 7808{
eb95419b 7809 struct task_group *tg = css_tg(css);
68318b8e
SV
7810
7811 sched_destroy_group(tg);
7812}
7813
eb95419b 7814static void cpu_cgroup_css_offline(struct cgroup_subsys_state *css)
ace783b9 7815{
eb95419b 7816 struct task_group *tg = css_tg(css);
ace783b9
LZ
7817
7818 sched_offline_group(tg);
7819}
7820
eb95419b 7821static int cpu_cgroup_can_attach(struct cgroup_subsys_state *css,
bb9d97b6 7822 struct cgroup_taskset *tset)
68318b8e 7823{
bb9d97b6
TH
7824 struct task_struct *task;
7825
924f0d9a 7826 cgroup_taskset_for_each(task, tset) {
b68aa230 7827#ifdef CONFIG_RT_GROUP_SCHED
eb95419b 7828 if (!sched_rt_can_attach(css_tg(css), task))
bb9d97b6 7829 return -EINVAL;
b68aa230 7830#else
bb9d97b6
TH
7831 /* We don't support RT-tasks being in separate groups */
7832 if (task->sched_class != &fair_sched_class)
7833 return -EINVAL;
b68aa230 7834#endif
bb9d97b6 7835 }
be367d09
BB
7836 return 0;
7837}
68318b8e 7838
eb95419b 7839static void cpu_cgroup_attach(struct cgroup_subsys_state *css,
bb9d97b6 7840 struct cgroup_taskset *tset)
68318b8e 7841{
bb9d97b6
TH
7842 struct task_struct *task;
7843
924f0d9a 7844 cgroup_taskset_for_each(task, tset)
bb9d97b6 7845 sched_move_task(task);
68318b8e
SV
7846}
7847
eb95419b
TH
7848static void cpu_cgroup_exit(struct cgroup_subsys_state *css,
7849 struct cgroup_subsys_state *old_css,
7850 struct task_struct *task)
068c5cc5
PZ
7851{
7852 /*
7853 * cgroup_exit() is called in the copy_process() failure path.
7854 * Ignore this case since the task hasn't ran yet, this avoids
7855 * trying to poke a half freed task state from generic code.
7856 */
7857 if (!(task->flags & PF_EXITING))
7858 return;
7859
7860 sched_move_task(task);
7861}
7862
052f1dc7 7863#ifdef CONFIG_FAIR_GROUP_SCHED
182446d0
TH
7864static int cpu_shares_write_u64(struct cgroup_subsys_state *css,
7865 struct cftype *cftype, u64 shareval)
68318b8e 7866{
182446d0 7867 return sched_group_set_shares(css_tg(css), scale_load(shareval));
68318b8e
SV
7868}
7869
182446d0
TH
7870static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css,
7871 struct cftype *cft)
68318b8e 7872{
182446d0 7873 struct task_group *tg = css_tg(css);
68318b8e 7874
c8b28116 7875 return (u64) scale_load_down(tg->shares);
68318b8e 7876}
ab84d31e
PT
7877
7878#ifdef CONFIG_CFS_BANDWIDTH
a790de99
PT
7879static DEFINE_MUTEX(cfs_constraints_mutex);
7880
ab84d31e
PT
7881const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */
7882const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */
7883
a790de99
PT
7884static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime);
7885
ab84d31e
PT
7886static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
7887{
56f570e5 7888 int i, ret = 0, runtime_enabled, runtime_was_enabled;
029632fb 7889 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
ab84d31e
PT
7890
7891 if (tg == &root_task_group)
7892 return -EINVAL;
7893
7894 /*
7895 * Ensure we have at some amount of bandwidth every period. This is
7896 * to prevent reaching a state of large arrears when throttled via
7897 * entity_tick() resulting in prolonged exit starvation.
7898 */
7899 if (quota < min_cfs_quota_period || period < min_cfs_quota_period)
7900 return -EINVAL;
7901
7902 /*
7903 * Likewise, bound things on the otherside by preventing insane quota
7904 * periods. This also allows us to normalize in computing quota
7905 * feasibility.
7906 */
7907 if (period > max_cfs_quota_period)
7908 return -EINVAL;
7909
0e59bdae
KT
7910 /*
7911 * Prevent race between setting of cfs_rq->runtime_enabled and
7912 * unthrottle_offline_cfs_rqs().
7913 */
7914 get_online_cpus();
a790de99
PT
7915 mutex_lock(&cfs_constraints_mutex);
7916 ret = __cfs_schedulable(tg, period, quota);
7917 if (ret)
7918 goto out_unlock;
7919
58088ad0 7920 runtime_enabled = quota != RUNTIME_INF;
56f570e5 7921 runtime_was_enabled = cfs_b->quota != RUNTIME_INF;
1ee14e6c
BS
7922 /*
7923 * If we need to toggle cfs_bandwidth_used, off->on must occur
7924 * before making related changes, and on->off must occur afterwards
7925 */
7926 if (runtime_enabled && !runtime_was_enabled)
7927 cfs_bandwidth_usage_inc();
ab84d31e
PT
7928 raw_spin_lock_irq(&cfs_b->lock);
7929 cfs_b->period = ns_to_ktime(period);
7930 cfs_b->quota = quota;
58088ad0 7931
a9cf55b2 7932 __refill_cfs_bandwidth_runtime(cfs_b);
58088ad0
PT
7933 /* restart the period timer (if active) to handle new period expiry */
7934 if (runtime_enabled && cfs_b->timer_active) {
7935 /* force a reprogram */
09dc4ab0 7936 __start_cfs_bandwidth(cfs_b, true);
58088ad0 7937 }
ab84d31e
PT
7938 raw_spin_unlock_irq(&cfs_b->lock);
7939
0e59bdae 7940 for_each_online_cpu(i) {
ab84d31e 7941 struct cfs_rq *cfs_rq = tg->cfs_rq[i];
029632fb 7942 struct rq *rq = cfs_rq->rq;
ab84d31e
PT
7943
7944 raw_spin_lock_irq(&rq->lock);
58088ad0 7945 cfs_rq->runtime_enabled = runtime_enabled;
ab84d31e 7946 cfs_rq->runtime_remaining = 0;
671fd9da 7947
029632fb 7948 if (cfs_rq->throttled)
671fd9da 7949 unthrottle_cfs_rq(cfs_rq);
ab84d31e
PT
7950 raw_spin_unlock_irq(&rq->lock);
7951 }
1ee14e6c
BS
7952 if (runtime_was_enabled && !runtime_enabled)
7953 cfs_bandwidth_usage_dec();
a790de99
PT
7954out_unlock:
7955 mutex_unlock(&cfs_constraints_mutex);
0e59bdae 7956 put_online_cpus();
ab84d31e 7957
a790de99 7958 return ret;
ab84d31e
PT
7959}
7960
7961int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us)
7962{
7963 u64 quota, period;
7964
029632fb 7965 period = ktime_to_ns(tg->cfs_bandwidth.period);
ab84d31e
PT
7966 if (cfs_quota_us < 0)
7967 quota = RUNTIME_INF;
7968 else
7969 quota = (u64)cfs_quota_us * NSEC_PER_USEC;
7970
7971 return tg_set_cfs_bandwidth(tg, period, quota);
7972}
7973
7974long tg_get_cfs_quota(struct task_group *tg)
7975{
7976 u64 quota_us;
7977
029632fb 7978 if (tg->cfs_bandwidth.quota == RUNTIME_INF)
ab84d31e
PT
7979 return -1;
7980
029632fb 7981 quota_us = tg->cfs_bandwidth.quota;
ab84d31e
PT
7982 do_div(quota_us, NSEC_PER_USEC);
7983
7984 return quota_us;
7985}
7986
7987int tg_set_cfs_period(struct task_group *tg, long cfs_period_us)
7988{
7989 u64 quota, period;
7990
7991 period = (u64)cfs_period_us * NSEC_PER_USEC;
029632fb 7992 quota = tg->cfs_bandwidth.quota;
ab84d31e 7993
ab84d31e
PT
7994 return tg_set_cfs_bandwidth(tg, period, quota);
7995}
7996
7997long tg_get_cfs_period(struct task_group *tg)
7998{
7999 u64 cfs_period_us;
8000
029632fb 8001 cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period);
ab84d31e
PT
8002 do_div(cfs_period_us, NSEC_PER_USEC);
8003
8004 return cfs_period_us;
8005}
8006
182446d0
TH
8007static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css,
8008 struct cftype *cft)
ab84d31e 8009{
182446d0 8010 return tg_get_cfs_quota(css_tg(css));
ab84d31e
PT
8011}
8012
182446d0
TH
8013static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css,
8014 struct cftype *cftype, s64 cfs_quota_us)
ab84d31e 8015{
182446d0 8016 return tg_set_cfs_quota(css_tg(css), cfs_quota_us);
ab84d31e
PT
8017}
8018
182446d0
TH
8019static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css,
8020 struct cftype *cft)
ab84d31e 8021{
182446d0 8022 return tg_get_cfs_period(css_tg(css));
ab84d31e
PT
8023}
8024
182446d0
TH
8025static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css,
8026 struct cftype *cftype, u64 cfs_period_us)
ab84d31e 8027{
182446d0 8028 return tg_set_cfs_period(css_tg(css), cfs_period_us);
ab84d31e
PT
8029}
8030
a790de99
PT
8031struct cfs_schedulable_data {
8032 struct task_group *tg;
8033 u64 period, quota;
8034};
8035
8036/*
8037 * normalize group quota/period to be quota/max_period
8038 * note: units are usecs
8039 */
8040static u64 normalize_cfs_quota(struct task_group *tg,
8041 struct cfs_schedulable_data *d)
8042{
8043 u64 quota, period;
8044
8045 if (tg == d->tg) {
8046 period = d->period;
8047 quota = d->quota;
8048 } else {
8049 period = tg_get_cfs_period(tg);
8050 quota = tg_get_cfs_quota(tg);
8051 }
8052
8053 /* note: these should typically be equivalent */
8054 if (quota == RUNTIME_INF || quota == -1)
8055 return RUNTIME_INF;
8056
8057 return to_ratio(period, quota);
8058}
8059
8060static int tg_cfs_schedulable_down(struct task_group *tg, void *data)
8061{
8062 struct cfs_schedulable_data *d = data;
029632fb 8063 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
a790de99
PT
8064 s64 quota = 0, parent_quota = -1;
8065
8066 if (!tg->parent) {
8067 quota = RUNTIME_INF;
8068 } else {
029632fb 8069 struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth;
a790de99
PT
8070
8071 quota = normalize_cfs_quota(tg, d);
9c58c79a 8072 parent_quota = parent_b->hierarchical_quota;
a790de99
PT
8073
8074 /*
8075 * ensure max(child_quota) <= parent_quota, inherit when no
8076 * limit is set
8077 */
8078 if (quota == RUNTIME_INF)
8079 quota = parent_quota;
8080 else if (parent_quota != RUNTIME_INF && quota > parent_quota)
8081 return -EINVAL;
8082 }
9c58c79a 8083 cfs_b->hierarchical_quota = quota;
a790de99
PT
8084
8085 return 0;
8086}
8087
8088static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota)
8089{
8277434e 8090 int ret;
a790de99
PT
8091 struct cfs_schedulable_data data = {
8092 .tg = tg,
8093 .period = period,
8094 .quota = quota,
8095 };
8096
8097 if (quota != RUNTIME_INF) {
8098 do_div(data.period, NSEC_PER_USEC);
8099 do_div(data.quota, NSEC_PER_USEC);
8100 }
8101
8277434e
PT
8102 rcu_read_lock();
8103 ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data);
8104 rcu_read_unlock();
8105
8106 return ret;
a790de99 8107}
e8da1b18 8108
2da8ca82 8109static int cpu_stats_show(struct seq_file *sf, void *v)
e8da1b18 8110{
2da8ca82 8111 struct task_group *tg = css_tg(seq_css(sf));
029632fb 8112 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
e8da1b18 8113
44ffc75b
TH
8114 seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods);
8115 seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled);
8116 seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time);
e8da1b18
NR
8117
8118 return 0;
8119}
ab84d31e 8120#endif /* CONFIG_CFS_BANDWIDTH */
6d6bc0ad 8121#endif /* CONFIG_FAIR_GROUP_SCHED */
68318b8e 8122
052f1dc7 8123#ifdef CONFIG_RT_GROUP_SCHED
182446d0
TH
8124static int cpu_rt_runtime_write(struct cgroup_subsys_state *css,
8125 struct cftype *cft, s64 val)
6f505b16 8126{
182446d0 8127 return sched_group_set_rt_runtime(css_tg(css), val);
6f505b16
PZ
8128}
8129
182446d0
TH
8130static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css,
8131 struct cftype *cft)
6f505b16 8132{
182446d0 8133 return sched_group_rt_runtime(css_tg(css));
6f505b16 8134}
d0b27fa7 8135
182446d0
TH
8136static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css,
8137 struct cftype *cftype, u64 rt_period_us)
d0b27fa7 8138{
182446d0 8139 return sched_group_set_rt_period(css_tg(css), rt_period_us);
d0b27fa7
PZ
8140}
8141
182446d0
TH
8142static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css,
8143 struct cftype *cft)
d0b27fa7 8144{
182446d0 8145 return sched_group_rt_period(css_tg(css));
d0b27fa7 8146}
6d6bc0ad 8147#endif /* CONFIG_RT_GROUP_SCHED */
6f505b16 8148
fe5c7cc2 8149static struct cftype cpu_files[] = {
052f1dc7 8150#ifdef CONFIG_FAIR_GROUP_SCHED
fe5c7cc2
PM
8151 {
8152 .name = "shares",
f4c753b7
PM
8153 .read_u64 = cpu_shares_read_u64,
8154 .write_u64 = cpu_shares_write_u64,
fe5c7cc2 8155 },
052f1dc7 8156#endif
ab84d31e
PT
8157#ifdef CONFIG_CFS_BANDWIDTH
8158 {
8159 .name = "cfs_quota_us",
8160 .read_s64 = cpu_cfs_quota_read_s64,
8161 .write_s64 = cpu_cfs_quota_write_s64,
8162 },
8163 {
8164 .name = "cfs_period_us",
8165 .read_u64 = cpu_cfs_period_read_u64,
8166 .write_u64 = cpu_cfs_period_write_u64,
8167 },
e8da1b18
NR
8168 {
8169 .name = "stat",
2da8ca82 8170 .seq_show = cpu_stats_show,
e8da1b18 8171 },
ab84d31e 8172#endif
052f1dc7 8173#ifdef CONFIG_RT_GROUP_SCHED
6f505b16 8174 {
9f0c1e56 8175 .name = "rt_runtime_us",
06ecb27c
PM
8176 .read_s64 = cpu_rt_runtime_read,
8177 .write_s64 = cpu_rt_runtime_write,
6f505b16 8178 },
d0b27fa7
PZ
8179 {
8180 .name = "rt_period_us",
f4c753b7
PM
8181 .read_u64 = cpu_rt_period_read_uint,
8182 .write_u64 = cpu_rt_period_write_uint,
d0b27fa7 8183 },
052f1dc7 8184#endif
4baf6e33 8185 { } /* terminate */
68318b8e
SV
8186};
8187
073219e9 8188struct cgroup_subsys cpu_cgrp_subsys = {
92fb9748
TH
8189 .css_alloc = cpu_cgroup_css_alloc,
8190 .css_free = cpu_cgroup_css_free,
ace783b9
LZ
8191 .css_online = cpu_cgroup_css_online,
8192 .css_offline = cpu_cgroup_css_offline,
bb9d97b6
TH
8193 .can_attach = cpu_cgroup_can_attach,
8194 .attach = cpu_cgroup_attach,
068c5cc5 8195 .exit = cpu_cgroup_exit,
5577964e 8196 .legacy_cftypes = cpu_files,
68318b8e
SV
8197 .early_init = 1,
8198};
8199
052f1dc7 8200#endif /* CONFIG_CGROUP_SCHED */
d842de87 8201
b637a328
PM
8202void dump_cpu_task(int cpu)
8203{
8204 pr_info("Task dump for CPU %d:\n", cpu);
8205 sched_show_task(cpu_curr(cpu));
8206}
This page took 2.568531 seconds and 5 git commands to generate.