#include <linux/mm.h>

#include <linux/module.h>

#include <linux/nmi.h>

#include <linux/init.h>

#include <linux/uaccess.h>

#include <linux/highmem.h>

#include <asm/mmu_context.h>

#include <linux/interrupt.h>

#include <linux/capability.h>

#include <linux/completion.h>

#include <linux/kernel_stat.h>

#include <linux/debug_locks.h>

#include <linux/perf_event.h>

#include <linux/security.h>

#include <linux/notifier.h>

#include <linux/profile.h>

#include <linux/freezer.h>

#include <linux/vmalloc.h>

#include <linux/blkdev.h>

#include <linux/delay.h>

#include <linux/pid_namespace.h>

#include <linux/smp.h>

#include <linux/threads.h>

#include <linux/timer.h>

#include <linux/rcupdate.h>

#include <linux/cpu.h>

#include <linux/cpuset.h>

#include <linux/percpu.h>

#include <linux/proc_fs.h>

#include <linux/seq_file.h>

#include <linux/stop_machine.h>

#include <linux/sysctl.h>

#include <linux/syscalls.h>

#include <linux/times.h>

#include <linux/tsacct_kern.h>

#include <linux/kprobes.h>

#include <linux/delayacct.h>

#include <linux/unistd.h>

#include <linux/pagemap.h>

#include <linux/hrtimer.h>

#include <linux/tick.h>

#include <linux/debugfs.h>

#include <linux/ctype.h>

#include <linux/ftrace.h>

#include <linux/slab.h>

#include <asm/tlb.h>

#include <asm/irq_regs.h>

#include <asm/mutex.h>

#ifdef CONFIG_PARAVIRT

#include <asm/paravirt.h>

#include "sched_cpupri.h"

#include "workqueue_sched.h"

#include "sched_autogroup.h"

#define CREATE_TRACE_POINTS

#include <trace/events/sched.h>

#define NICE_TO_PRIO(nice) (MAX_RT_PRIO + (nice) + 20)

#define PRIO_TO_NICE(prio) ((prio) - MAX_RT_PRIO - 20)

#define TASK_NICE(p) PRIO_TO_NICE((p)->static_prio)

#define USER_PRIO(p) ((p)-MAX_RT_PRIO)

#define TASK_USER_PRIO(p) USER_PRIO((p)->static_prio)

#define MAX_USER_PRIO (USER_PRIO(MAX_PRIO))

#define NS_TO_JIFFIES(TIME) ((unsigned long)(TIME) / (NSEC_PER_SEC / HZ))

#define NICE_0_LOAD SCHED_LOAD_SCALE

#define NICE_0_SHIFT SCHED_LOAD_SHIFT

#define DEF_TIMESLICE (100 * HZ / 1000)

#define RUNTIME_INF ((u64)~0ULL)

static inline int rt_policy(int policy)

{

if (policy == SCHED_FIFO || policy == SCHED_RR)

return 1;

return 0;

}

static inline int task_has_rt_policy(struct task_struct *p)

{

return rt_policy(p->policy);

}

struct rt_prio_array {

DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */

struct list_head queue[MAX_RT_PRIO];

};

struct rt_bandwidth {

};

static struct rt_bandwidth def_rt_bandwidth;

static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);

static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)

{

struct rt_bandwidth *rt_b =

container_of(timer, struct rt_bandwidth, rt_period_timer);

ktime_t now;

int overrun;

int idle = 0;

for (;;) {

now = hrtimer_cb_get_time(timer);

overrun = hrtimer_forward(timer, now, rt_b->rt_period);

if (!overrun)

break;

idle = do_sched_rt_period_timer(rt_b, overrun);

}

return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;

}

void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)

{

rt_b->rt_period = ns_to_ktime(period);

rt_b->rt_runtime = runtime;

raw_spin_lock_init(&rt_b->rt_runtime_lock);

hrtimer_init(&rt_b->rt_period_timer,

CLOCK_MONOTONIC, HRTIMER_MODE_REL);

rt_b->rt_period_timer.function = sched_rt_period_timer;

}

static inline int rt_bandwidth_enabled(void)

{

return sysctl_sched_rt_runtime >= 0;

}

static void start_rt_bandwidth(struct rt_bandwidth *rt_b)

{

ktime_t now;

if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)

return;

if (hrtimer_active(&rt_b->rt_period_timer))

return;

raw_spin_lock(&rt_b->rt_runtime_lock);

for (;;) {

unsigned long delta;

ktime_t soft, hard;

if (hrtimer_active(&rt_b->rt_period_timer))

break;

now = hrtimer_cb_get_time(&rt_b->rt_period_timer);

hrtimer_forward(&rt_b->rt_period_timer, now, rt_b->rt_period);

soft = hrtimer_get_softexpires(&rt_b->rt_period_timer);

hard = hrtimer_get_expires(&rt_b->rt_period_timer);

delta = ktime_to_ns(ktime_sub(hard, soft));

__hrtimer_start_range_ns(&rt_b->rt_period_timer, soft, delta,

HRTIMER_MODE_ABS_PINNED, 0);

}

raw_spin_unlock(&rt_b->rt_runtime_lock);

}

#ifdef CONFIG_RT_GROUP_SCHED

static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)

{

hrtimer_cancel(&rt_b->rt_period_timer);

}

static DEFINE_MUTEX(sched_domains_mutex);

#ifdef CONFIG_CGROUP_SCHED

#include <linux/cgroup.h>

struct cfs_rq;

static LIST_HEAD(task_groups);

struct task_group {

};

static DEFINE_SPINLOCK(task_group_lock);

#ifdef CONFIG_FAIR_GROUP_SCHED

# define ROOT_TASK_GROUP_LOAD NICE_0_LOAD

#define MIN_SHARES (1UL << 1)

#define MAX_SHARES (1UL << 18)

static int root_task_group_load = ROOT_TASK_GROUP_LOAD;

struct task_group root_task_group;

#endif /* CONFIG_CGROUP_SCHED */

struct cfs_rq {

};

struct rt_rq {

};

#ifdef CONFIG_SMP

struct root_domain {

};

static struct root_domain def_root_domain;

#endif /* CONFIG_SMP */

struct rq {

};

static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);

static void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags);

static inline int cpu_of(struct rq *rq)

{

#ifdef CONFIG_SMP

return rq->cpu;

return 0;

}

#define rcu_dereference_check_sched_domain(p) \

rcu_dereference_check((p), \

lockdep_is_held(&sched_domains_mutex))

#define for_each_domain(cpu, __sd) \

for (__sd = rcu_dereference_check_sched_domain(cpu_rq(cpu)->sd); __sd; __sd = __sd->parent)

#define cpu_rq(cpu) (&per_cpu(runqueues, (cpu)))

#define this_rq() (&__get_cpu_var(runqueues))

#define task_rq(p) cpu_rq(task_cpu(p))

#define cpu_curr(cpu) (cpu_rq(cpu)->curr)

#define raw_rq() (&__raw_get_cpu_var(runqueues))

#ifdef CONFIG_CGROUP_SCHED

static inline struct task_group *task_group(struct task_struct *p)

{

struct task_group *tg;

struct cgroup_subsys_state *css;

css = task_subsys_state_check(p, cpu_cgroup_subsys_id,

lockdep_is_held(&p->pi_lock) ||

lockdep_is_held(&task_rq(p)->lock));

tg = container_of(css, struct task_group, css);

return autogroup_task_group(p, tg);

}

static inline void set_task_rq(struct task_struct *p, unsigned int cpu)

{

#ifdef CONFIG_FAIR_GROUP_SCHED

p->se.cfs_rq = task_group(p)->cfs_rq[cpu];

p->se.parent = task_group(p)->se[cpu];

#ifdef CONFIG_RT_GROUP_SCHED

p->rt.rt_rq = task_group(p)->rt_rq[cpu];

p->rt.parent = task_group(p)->rt_se[cpu];

}

#else /* CONFIG_CGROUP_SCHED */

static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }

static inline struct task_group *task_group(struct task_struct *p)

{

return NULL;

}

#endif /* CONFIG_CGROUP_SCHED */

static void update_rq_clock_task(struct rq *rq, s64 delta);

static void update_rq_clock(struct rq *rq)

{

s64 delta;

if (rq->skip_clock_update > 0)

return;

delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;

rq->clock += delta;

update_rq_clock_task(rq, delta);

}

#ifdef CONFIG_SCHED_DEBUG

# define const_debug __read_mostly

# define const_debug static const

int runqueue_is_locked(int cpu)

{

return raw_spin_is_locked(&cpu_rq(cpu)->lock);

}

#define SCHED_FEAT(name, enabled) \

__SCHED_FEAT_##name ,

enum {

#include "sched_features.h"

};

#undef SCHED_FEAT

#define SCHED_FEAT(name, enabled) \

(1UL << __SCHED_FEAT_##name) * enabled |

const_debug unsigned int sysctl_sched_features =

#include "sched_features.h"

0;

#undef SCHED_FEAT

#ifdef CONFIG_SCHED_DEBUG

#define SCHED_FEAT(name, enabled) \

#name ,

static __read_mostly char *sched_feat_names[] = {

#include "sched_features.h"

NULL

};

#undef SCHED_FEAT

static int sched_feat_show(struct seq_file *m, void *v)

{

int i;

for (i = 0; sched_feat_names[i]; i++) {

if (!(sysctl_sched_features & (1UL << i)))

seq_puts(m, "NO_");

seq_printf(m, "%s ", sched_feat_names[i]);

}

seq_puts(m, "\n");

return 0;

}

static ssize_t

sched_feat_write(struct file *filp, const char __user *ubuf,

size_t cnt, loff_t *ppos)

{

char buf[64];

char *cmp;

int neg = 0;

int i;

if (cnt > 63)

cnt = 63;

if (copy_from_user(&buf, ubuf, cnt))

return -EFAULT;

buf[cnt] = 0;

cmp = strstrip(buf);

if (strncmp(cmp, "NO_", 3) == 0) {

neg = 1;

cmp += 3;

}

for (i = 0; sched_feat_names[i]; i++) {

if (strcmp(cmp, sched_feat_names[i]) == 0) {

if (neg)

sysctl_sched_features &= ~(1UL << i);

else

sysctl_sched_features |= (1UL << i);

break;

}

}

if (!sched_feat_names[i])

return -EINVAL;

*ppos += cnt;

return cnt;

}

static int sched_feat_open(struct inode *inode, struct file *filp)

{

return single_open(filp, sched_feat_show, NULL);

}

static const struct file_operations sched_feat_fops = {

.open = sched_feat_open,

.write = sched_feat_write,

.read = seq_read,

.llseek = seq_lseek,

.release = single_release,

};

static __init int sched_init_debug(void)

{

debugfs_create_file("sched_features", 0644, NULL, NULL,

&sched_feat_fops);

return 0;

}

late_initcall(sched_init_debug);

#define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))

const_debug unsigned int sysctl_sched_nr_migrate = 32;

const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC;

unsigned int sysctl_sched_rt_period = 1000000;

static __read_mostly int scheduler_running;

int sysctl_sched_rt_runtime = 950000;

static inline u64 global_rt_period(void)

{

return (u64)sysctl_sched_rt_period * NSEC_PER_USEC;

}

static inline u64 global_rt_runtime(void)

{

if (sysctl_sched_rt_runtime < 0)

return RUNTIME_INF;

return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC;

}

#ifndef prepare_arch_switch

# define prepare_arch_switch(next) do { } while (0)

#ifndef finish_arch_switch

# define finish_arch_switch(prev) do { } while (0)

static inline int task_current(struct rq *rq, struct task_struct *p)

{

return rq->curr == p;

}

static inline int task_running(struct rq *rq, struct task_struct *p)

{

#ifdef CONFIG_SMP

return p->on_cpu;

return task_current(rq, p);

}

#ifndef __ARCH_WANT_UNLOCKED_CTXSW

static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)

{

#ifdef CONFIG_SMP

/*

next->on_cpu = 1;

}

static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)

{

#ifdef CONFIG_SMP

/*

smp_wmb();

prev->on_cpu = 0;

#ifdef CONFIG_DEBUG_SPINLOCK

/* this is a valid case when another task releases the spinlock */

rq->lock.owner = current;

/*

spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);

raw_spin_unlock_irq(&rq->lock);

}

#else /* __ARCH_WANT_UNLOCKED_CTXSW */

static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)

{

#ifdef CONFIG_SMP

/*

next->on_cpu = 1;

#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW

raw_spin_unlock_irq(&rq->lock);

raw_spin_unlock(&rq->lock);

}

static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)

{

#ifdef CONFIG_SMP

/*

smp_wmb();

prev->on_cpu = 0;

#ifndef __ARCH_WANT_INTERRUPTS_ON_CTXSW

local_irq_enable();

}

#endif /* __ARCH_WANT_UNLOCKED_CTXSW */

static inline struct rq *__task_rq_lock(struct task_struct *p)

__acquires(rq->lock)

{

struct rq *rq;

lockdep_assert_held(&p->pi_lock);

for (;;) {

rq = task_rq(p);

raw_spin_lock(&rq->lock);

if (likely(rq == task_rq(p)))

return rq;

raw_spin_unlock(&rq->lock);

}

}

static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)

__acquires(p->pi_lock)

__acquires(rq->lock)

{

struct rq *rq;

for (;;) {

raw_spin_lock_irqsave(&p->pi_lock, *flags);

rq = task_rq(p);

raw_spin_lock(&rq->lock);

if (likely(rq == task_rq(p)))

return rq;

raw_spin_unlock(&rq->lock);

raw_spin_unlock_irqrestore(&p->pi_lock, *flags);

}

}

static void __task_rq_unlock(struct rq *rq)

__releases(rq->lock)

{

raw_spin_unlock(&rq->lock);

}

static inline void

task_rq_unlock(struct rq *rq, struct task_struct *p, unsigned long *flags)

__releases(rq->lock)

__releases(p->pi_lock)

{

raw_spin_unlock(&rq->lock);

raw_spin_unlock_irqrestore(&p->pi_lock, *flags);

}

static struct rq *this_rq_lock(void)

__acquires(rq->lock)