Index: linux-2.6.10-rc1-mm2-plugsched1/fs/proc/array.c =================================================================== --- linux-2.6.10-rc1-mm2-plugsched1.orig/fs/proc/array.c 2004-11-01 14:50:17.000000000 +1100 +++ linux-2.6.10-rc1-mm2-plugsched1/fs/proc/array.c 2004-11-01 14:50:38.000000000 +1100 @@ -162,7 +162,6 @@ static inline char * task_state(struct t read_lock(&tasklist_lock); buffer += sprintf(buffer, "State:\t%s\n" - "SleepAVG:\t%lu%%\n" "Tgid:\t%d\n" "Pid:\t%d\n" "PPid:\t%d\n" @@ -170,7 +169,6 @@ static inline char * task_state(struct t "Uid:\t%d\t%d\t%d\t%d\n" "Gid:\t%d\t%d\t%d\t%d\n", get_task_state(p), - (p->sleep_avg/1024)*100/(1020000000/1024), p->tgid, p->pid, p->pid ? p->group_leader->real_parent->tgid : 0, p->pid && p->ptrace ? p->parent->pid : 0, Index: linux-2.6.10-rc1-mm2-plugsched1/fs/proc/proc_misc.c =================================================================== --- linux-2.6.10-rc1-mm2-plugsched1.orig/fs/proc/proc_misc.c 2004-11-01 14:50:16.000000000 +1100 +++ linux-2.6.10-rc1-mm2-plugsched1/fs/proc/proc_misc.c 2004-11-01 14:50:39.000000000 +1100 @@ -45,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -265,6 +266,18 @@ static int version_read_proc(char *page, return proc_calc_metrics(page, start, off, count, eof, len); } +static int scheduler_read_proc(char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + char *sched_name = scheduler->cpusched_name; + int len; + + strcpy(page, sched_name); + strcat(page, "\n"); + len = strlen(page); + return proc_calc_metrics(page, start, off, count, eof, len); +} + extern struct seq_operations cpuinfo_op; static int cpuinfo_open(struct inode *inode, struct file *file) { @@ -608,6 +621,7 @@ void __init proc_misc_init(void) {"cmdline", cmdline_read_proc}, {"locks", locks_read_proc}, {"execdomains", execdomains_read_proc}, + {"scheduler", scheduler_read_proc}, {NULL,} }; for (p = simple_ones; p->name; p++) Index: linux-2.6.10-rc1-mm2-plugsched1/include/linux/init_task.h =================================================================== --- linux-2.6.10-rc1-mm2-plugsched1.orig/include/linux/init_task.h 2004-11-01 14:50:17.000000000 +1100 +++ linux-2.6.10-rc1-mm2-plugsched1/include/linux/init_task.h 2004-11-01 14:50:38.000000000 +1100 @@ -72,14 +72,10 @@ extern struct group_info init_groups; .usage = ATOMIC_INIT(2), \ .flags = 0, \ .lock_depth = -1, \ - .prio = MAX_PRIO-20, \ - .static_prio = MAX_PRIO-20, \ .policy = SCHED_NORMAL, \ .cpus_allowed = CPU_MASK_ALL, \ .mm = NULL, \ .active_mm = &init_mm, \ - .run_list = LIST_HEAD_INIT(tsk.run_list), \ - .time_slice = HZ, \ .tasks = LIST_HEAD_INIT(tsk.tasks), \ .ptrace_children= LIST_HEAD_INIT(tsk.ptrace_children), \ .ptrace_list = LIST_HEAD_INIT(tsk.ptrace_list), \ @@ -115,5 +111,4 @@ extern struct group_info init_groups; .private_pages = LIST_HEAD_INIT(tsk.private_pages), \ .private_pages_count = 0, \ } - #endif Index: linux-2.6.10-rc1-mm2-plugsched1/include/linux/sched.h =================================================================== --- linux-2.6.10-rc1-mm2-plugsched1.orig/include/linux/sched.h 2004-11-01 14:50:19.000000000 +1100 +++ linux-2.6.10-rc1-mm2-plugsched1/include/linux/sched.h 2004-11-01 14:50:39.329524652 +1100 @@ -32,6 +32,7 @@ #include #include #include +#include struct exec_domain; @@ -165,9 +166,6 @@ extern void show_regs(struct pt_regs *); */ extern void show_stack(struct task_struct *task, unsigned long *sp); -void io_schedule(void); -long io_schedule_timeout(long timeout); - extern void cpu_init (void); extern void trap_init(void); extern void update_process_times(int user); @@ -179,6 +177,9 @@ extern unsigned long cache_decay_ticks; /* Is this address in the __sched functions? */ extern int in_sched_functions(unsigned long addr); +void __sched io_schedule(void); +long __sched io_schedule_timeout(long timeout); + #define MAX_SCHEDULE_TIMEOUT LONG_MAX extern signed long FASTCALL(schedule_timeout(signed long timeout)); asmlinkage void schedule(void); @@ -328,11 +329,6 @@ struct signal_struct { }; /* - * Priority of a process goes from 0..MAX_PRIO-1, valid RT - * priority is 0..MAX_RT_PRIO-1, and SCHED_NORMAL tasks are - * in the range MAX_RT_PRIO..MAX_PRIO-1. Priority values - * are inverted: lower p->prio value means higher priority. - * * The MAX_USER_RT_PRIO value allows the actual maximum * RT priority to be separate from the value exported to * user-space. This allows kernel threads to set their @@ -343,9 +339,7 @@ struct signal_struct { #define MAX_USER_RT_PRIO 100 #define MAX_RT_PRIO MAX_USER_RT_PRIO -#define MAX_PRIO (MAX_RT_PRIO + 40) - -#define rt_task(p) (unlikely((p)->prio < MAX_RT_PRIO)) +extern int rt_task(task_t *p); /* * Some day this will be a full-fledged user tracking system.. @@ -463,13 +457,11 @@ struct sched_domain { #endif }; -#ifdef ARCH_HAS_SCHED_DOMAIN /* Useful helpers that arch setup code may use. Defined in kernel/sched.c */ -extern cpumask_t cpu_isolated_map; +extern void cpu_attach_domain(struct sched_domain *sd, int cpu); extern void init_sched_build_groups(struct sched_group groups[], cpumask_t span, int (*group_fn)(int cpu)); -extern void cpu_attach_domain(struct sched_domain *sd, int cpu); -#endif /* ARCH_HAS_SCHED_DOMAIN */ +extern cpumask_t cpu_isolated_map; #endif /* CONFIG_SMP */ @@ -513,6 +505,10 @@ int set_current_groups(struct group_info struct audit_context; /* See audit.c */ struct mempolicy; +#include + +extern struct sched_drv *scheduler; + struct task_struct { volatile long state; /* -1 unrunnable, 0 runnable, >0 stopped */ struct thread_info *thread_info; @@ -522,18 +518,11 @@ struct task_struct { int lock_depth; /* Lock depth */ - int prio, static_prio; - struct list_head run_list; - prio_array_t *array; - - unsigned long sleep_avg; - long interactive_credit; - unsigned long long timestamp, last_ran; - int activated; + int static_prio; /* A commonality between cpu schedulers */ + union cpusched u; unsigned long policy; cpumask_t cpus_allowed; - unsigned int time_slice, first_time_slice; #ifdef CONFIG_SCHEDSTATS struct sched_info sched_info; @@ -739,6 +728,7 @@ extern int task_prio(const task_t *p); extern int task_nice(const task_t *p); extern int task_curr(const task_t *p); extern int idle_cpu(int cpu); +extern void set_oom_timeslice(task_t *p); void yield(void); @@ -764,6 +754,7 @@ static inline int kstack_end(void *addr) extern union thread_union init_thread_union; extern struct task_struct init_task; +extern struct task_struct base_init_task; extern struct mm_struct init_mm; @@ -1085,33 +1076,8 @@ extern void recalc_sigpending(void); extern void signal_wake_up(struct task_struct *t, int resume_stopped); -/* - * Wrappers for p->thread_info->cpu access. No-op on UP. - */ -#ifdef CONFIG_SMP - -static inline unsigned int task_cpu(const struct task_struct *p) -{ - return p->thread_info->cpu; -} - -static inline void set_task_cpu(struct task_struct *p, unsigned int cpu) -{ - p->thread_info->cpu = cpu; -} - -#else - -static inline unsigned int task_cpu(const struct task_struct *p) -{ - return 0; -} - -static inline void set_task_cpu(struct task_struct *p, unsigned int cpu) -{ -} - -#endif /* CONFIG_SMP */ +extern unsigned int task_cpu(const struct task_struct *p); +extern void set_task_cpu(struct task_struct *p, unsigned int cpu); #ifdef HAVE_ARCH_PICK_MMAP_LAYOUT extern void arch_pick_mmap_layout(struct mm_struct *mm); Index: linux-2.6.10-rc1-mm2-plugsched1/include/linux/scheduler.h =================================================================== --- linux-2.6.10-rc1-mm2-plugsched1.orig/include/linux/scheduler.h 2003-03-27 19:01:40.000000000 +1100 +++ linux-2.6.10-rc1-mm2-plugsched1/include/linux/scheduler.h 2004-11-01 14:50:39.330524493 +1100 @@ -0,0 +1,108 @@ +/* + * include/linux/scheduler.h + * This contains the driver struct for all the exported per-cpu-scheduler + * functions, and the private per-scheduler data in task_struct. + */ +#define SCHED_NAME_MAX (16) + +/* + * This is the main scheduler driver struct. + */ +struct sched_drv +{ + unsigned int (*task_cpu)(const struct task_struct *); + void (*set_task_cpu)(struct task_struct *, unsigned int); + void (*init_sched_domain_sysctl)(void); + void (*destroy_sched_domain_sysctl)(void); + void (*account_steal_time)(struct task_struct *, cputime_t); + void (*account_system_time)(struct task_struct *, int, cputime_t); + void (*account_user_time)(struct task_struct *, cputime_t); + char cpusched_name[SCHED_NAME_MAX]; + int (*rt_task)(task_t *); + void (*wait_for_completion)(struct completion *); + void (*io_schedule)(void); + long (*io_schedule_timeout)(long); + void (*sched_idle_next)(void); + void (*set_oom_timeslice)(task_t *); + unsigned long (*nr_running)(void); + unsigned long (*nr_uninterruptible)(void); + unsigned long long (*nr_context_switches)(void); + unsigned long (*nr_iowait)(void); + int (*idle_cpu)(int); + void (*init_idle)(task_t *, int); + void (*exit)(task_t *); + void (*fork)(task_t *); + void (*init)(void); + void (*init_smp)(void); + void (*schedule)(void); + void (*tick)(void); + void (*tail)(task_t *); + int (*setscheduler)(pid_t, int, struct sched_param __user *); + void (*set_user_nice)(task_t *, long); + long (*rr_get_interval)(pid_t, struct timespec __user *); + long (*yield)(void); + int (*task_curr)(const task_t *); + int (*task_nice)(const task_t *); + int (*task_prio)(const task_t *); + int (*try_to_wake_up)(task_t *, unsigned, int); + void (*wake_up_new_task)(task_t *, unsigned long); +#ifdef CONFIG_SMP + int (*migration_init)(void); + void (*exec)(void); + int (*set_cpus_allowed)(task_t *, cpumask_t); + void (*wait_task_inactive)(task_t *); + void (*cpu_attach_domain)(struct sched_domain *, int); +#endif +#ifdef CONFIG_SCHEDSTATS + int (*show_schedstat)(struct seq_file *, void *); +#endif +}; + +/* + * List functions that have common variants that many schedulers use. + */ +extern unsigned int common_task_cpu(const struct task_struct *p); +extern void common_set_task_cpu(struct task_struct *p, unsigned int cpu); + +/* + * All private per-scheduler entries in task_struct are defined here as + * separate structs placed into the cpusched union in task_struct. + */ + +/* Ingosched */ +#ifdef CONFIG_CPUSCHED_INGO +struct cpusched_ingo { + int prio; + struct list_head run_list; + prio_array_t *array; + unsigned int time_slice; + unsigned int first_time_slice; + unsigned long sleep_avg; + long interactive_credit; + unsigned long timestamp; + unsigned long long last_ran; + int activated; +}; +#endif + +/* Staircase scheduler */ +#ifdef CONFIG_CPUSCHED_STAIRCASE +struct cpusched_sc { + int prio; + struct list_head run_list; + unsigned long sflags; + unsigned long long timestamp; + unsigned long runtime, totalrun, ns_debit; + unsigned int burst; + unsigned int slice, time_slice; +}; +#endif + +union cpusched { +#ifdef CONFIG_CPUSCHED_INGO + struct cpusched_ingo ingosched; +#endif +#ifdef CONFIG_CPUSCHED_STAIRCASE + struct cpusched_sc scsched; +#endif +}; Index: linux-2.6.10-rc1-mm2-plugsched1/init/Kconfig =================================================================== --- linux-2.6.10-rc1-mm2-plugsched1.orig/init/Kconfig 2004-11-01 14:50:16.000000000 +1100 +++ linux-2.6.10-rc1-mm2-plugsched1/init/Kconfig 2004-11-01 14:50:39.000000000 +1100 @@ -249,6 +249,40 @@ config IKCONFIG_PROC through /proc/config.gz. +config PLUGSCHED + bool "Support for multiple cpu schedulers" + default y + help + Say Y here if you want to compile in support for multiple + cpu schedulers. The cpu scheduler may be selected at boot time + with the boot parameter "cpusched=". The choice of which cpu + schedulers to compile into the kernel can be made by enabling + "Configure standard kernel features" otherwise all cpu schedulers + supported will be compiled in. + +choice + prompt "Default cpu scheduler" + help + This option allows you to choose which cpu scheduler shall be + booted by default at startup if you have plugsched support, or + it will choose which is the only scheduler compiled in. + +config CPUSCHED_DEFAULT_INGO + bool "Ingosched cpu scheduler" + select CPUSCHED_INGO + ---help--- + This is the default cpu scheduler which is an O(1) dual priority + array scheduler with a hybrid interactive design. + +config CPUSCHED_DEFAULT_STAIRCASE + bool "Staircase cpu scheduler" + select CPUSCHED_STAIRCASE + ---help--- + This scheduler is an O(1) single priority array with a foreground- + background interactive design. + +endchoice + menuconfig EMBEDDED bool "Configure standard kernel features (for small systems)" help @@ -257,6 +291,26 @@ menuconfig EMBEDDED environments which can tolerate a "non-standard" kernel. Only use this if you really know what you are doing. +config CPUSCHED_INGO + bool "Ingosched cpu scheduler" if EMBEDDED + depends on PLUGSCHED + default y + ---help--- + This is the default cpu scheduler which is an O(1) dual priority + array scheduler with a hybrid interactive design. + To boot this cpu scheduler, if it is not the default, use the + bootparam "cpusched=ingosched". + +config CPUSCHED_STAIRCASE + bool "Staircase cpu scheduler" if EMBEDDED + depends on PLUGSCHED + default y + ---help--- + This scheduler is an O(1) single priority array with a foreground- + background interactive design. + To boot this cpu scheduler, if it is not the default, use the + bootparam "cpusched=staircase". + config KALLSYMS bool "Load all symbols for debugging/kksymoops" if EMBEDDED default y Index: linux-2.6.10-rc1-mm2-plugsched1/init/main.c =================================================================== --- linux-2.6.10-rc1-mm2-plugsched1.orig/init/main.c 2004-11-01 14:50:16.000000000 +1100 +++ linux-2.6.10-rc1-mm2-plugsched1/init/main.c 2004-11-01 14:50:39.070565797 +1100 @@ -47,6 +47,7 @@ #include #include #include +#include #include #include @@ -487,10 +488,11 @@ void __init parse_early_param(void) done = 1; } +struct task_struct base_init_task; + /* * Activate the first processor. */ - asmlinkage void __init start_kernel(void) { char * command_line; @@ -512,6 +514,11 @@ asmlinkage void __init start_kernel(void smp_prepare_boot_cpu(); /* + * Save a copy of the baseline init_task in case we need to start + * another cpu scheduler. + */ + base_init_task = init_task; + /* * Set up the scheduler prior starting any interrupts (such as the * timer interrupt). Full topology setup happens at smp_init() * time - but meanwhile we still have a functioning scheduler. @@ -590,6 +597,7 @@ asmlinkage void __init start_kernel(void acpi_early_init(); /* before LAPIC and SMP init */ + printk("Running with %s cpu scheduler.\n", scheduler->cpusched_name); /* Do the rest non-__init'ed, we're now alive */ rest_init(); } Index: linux-2.6.10-rc1-mm2-plugsched1/kernel/Makefile =================================================================== --- linux-2.6.10-rc1-mm2-plugsched1.orig/kernel/Makefile 2004-11-01 14:50:17.000000000 +1100 +++ linux-2.6.10-rc1-mm2-plugsched1/kernel/Makefile 2004-11-01 14:50:39.000000000 +1100 @@ -2,13 +2,15 @@ # Makefile for the linux kernel. # -obj-y = sched.o fork.o exec_domain.o panic.o printk.o profile.o \ +obj-y = scheduler.o fork.o exec_domain.o panic.o printk.o profile.o \ exit.o itimer.o time.o softirq.o resource.o \ sysctl.o capability.o ptrace.o timer.o user.o \ signal.o sys.o kmod.o workqueue.o pid.o \ rcupdate.o intermodule.o extable.o params.o posix-timers.o \ kthread.o wait.o kfifo.o +obj-$(CONFIG_CPUSCHED_INGO) += sched.o +obj-$(CONFIG_CPUSCHED_STAIRCASE) += staircase.o obj-$(CONFIG_FUTEX) += futex.o obj-$(CONFIG_GENERIC_ISA_DMA) += dma.o obj-$(CONFIG_SMP) += cpu.o spinlock.o Index: linux-2.6.10-rc1-mm2-plugsched1/kernel/sched.c =================================================================== --- linux-2.6.10-rc1-mm2-plugsched1.orig/kernel/sched.c 2004-11-01 14:50:19.000000000 +1100 +++ linux-2.6.10-rc1-mm2-plugsched1/kernel/sched.c 2004-11-01 14:50:39.000000000 +1100 @@ -1,7 +1,7 @@ /* * kernel/sched.c * - * Kernel scheduler and related syscalls + * This is "ingosched"; the default cpu scheduler. * * Copyright (C) 1991-2002 Linus Torvalds * @@ -47,6 +47,7 @@ #include #include #include +#include #include #include @@ -58,6 +59,15 @@ #endif /* + * Priority of a process goes from 0..MAX_PRIO-1, valid RT + * priority is 0..MAX_RT_PRIO-1, and SCHED_NORMAL tasks are + * in the range MAX_RT_PRIO..MAX_PRIO-1. Priority values + * are inverted: lower p->prio value means higher priority. + */ + +#define MAX_PRIO (MAX_RT_PRIO + 40) + +/* * Convert user-nice values [ -20 ... 0 ... 19 ] * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ], * and back. @@ -131,7 +141,7 @@ */ #define CURRENT_BONUS(p) \ - (NS_TO_JIFFIES((p)->sleep_avg) * MAX_BONUS / \ + (NS_TO_JIFFIES((p)->u.ingosched.sleep_avg) * MAX_BONUS / \ MAX_SLEEP_AVG) #ifdef CONFIG_SMP @@ -150,20 +160,20 @@ (SCALE(TASK_NICE(p), 40, MAX_BONUS) + INTERACTIVE_DELTA) #define TASK_INTERACTIVE(p) \ - ((p)->prio <= (p)->static_prio - DELTA(p)) + ((p)->u.ingosched.prio <= (p)->static_prio - DELTA(p)) #define INTERACTIVE_SLEEP(p) \ (JIFFIES_TO_NS(MAX_SLEEP_AVG * \ (MAX_BONUS / 2 + DELTA((p)) + 1) / MAX_BONUS - 1)) #define HIGH_CREDIT(p) \ - ((p)->interactive_credit > CREDIT_LIMIT) + ((p)->u.ingosched.interactive_credit > CREDIT_LIMIT) #define LOW_CREDIT(p) \ - ((p)->interactive_credit < -CREDIT_LIMIT) + ((p)->u.ingosched.interactive_credit < -CREDIT_LIMIT) #define TASK_PREEMPTS_CURR(p, rq) \ - ((p)->prio < (rq)->curr->prio) + ((p)->u.ingosched.prio < (rq)->curr->u.ingosched.prio) /* * task_timeslice() scales user-nice values [ -20 ... 0 ... 19 ] @@ -184,7 +194,7 @@ static unsigned int task_timeslice(task_ else return SCALE_PRIO(DEF_TIMESLICE, p->static_prio); } -#define task_hot(p, now, sd) ((long long) ((now) - (p)->last_ran) \ +#define task_hot(p, now, sd) ((long long) ((now) - (p)->u.ingosched.last_ran) \ < (long long) (sd)->cache_hot_time) /* @@ -292,6 +302,11 @@ static DEFINE_PER_CPU(struct runqueue, r #define task_rq(p) cpu_rq(task_cpu(p)) #define cpu_curr(cpu) (cpu_rq(cpu)->curr) +static int ingo_rt_task(task_t *p) +{ + return (unlikely((p)->u.ingosched.prio < MAX_RT_PRIO)); +} + /* * Default context-switch locking: */ @@ -333,7 +348,7 @@ static inline void task_rq_unlock(runque */ #define SCHEDSTAT_VERSION 10 -static int show_schedstat(struct seq_file *seq, void *v) +static int ingo_show_schedstat(struct seq_file *seq, void *v) { int cpu; enum idle_type itype; @@ -392,32 +407,6 @@ static int show_schedstat(struct seq_fil return 0; } -static int schedstat_open(struct inode *inode, struct file *file) -{ - unsigned int size = PAGE_SIZE * (1 + num_online_cpus() / 32); - char *buf = kmalloc(size, GFP_KERNEL); - struct seq_file *m; - int res; - - if (!buf) - return -ENOMEM; - res = single_open(file, show_schedstat, NULL); - if (!res) { - m = file->private_data; - m->buf = buf; - m->size = size; - } else - kfree(buf); - return res; -} - -struct file_operations proc_schedstat_operations = { - .open = schedstat_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; - # define schedstat_inc(rq, field) rq->field++; # define schedstat_add(rq, field, amt) rq->field += amt; #else /* !CONFIG_SCHEDSTATS */ @@ -572,18 +561,18 @@ static inline void sched_info_switch(tas static void dequeue_task(struct task_struct *p, prio_array_t *array) { array->nr_active--; - list_del(&p->run_list); - if (list_empty(array->queue + p->prio)) - __clear_bit(p->prio, array->bitmap); + list_del(&p->u.ingosched.run_list); + if (list_empty(array->queue + p->u.ingosched.prio)) + __clear_bit(p->u.ingosched.prio, array->bitmap); } static void enqueue_task(struct task_struct *p, prio_array_t *array) { sched_info_queued(p); - list_add_tail(&p->run_list, array->queue + p->prio); - __set_bit(p->prio, array->bitmap); + list_add_tail(&p->u.ingosched.run_list, array->queue + p->u.ingosched.prio); + __set_bit(p->u.ingosched.prio, array->bitmap); array->nr_active++; - p->array = array; + p->u.ingosched.array = array; } /* @@ -593,10 +582,15 @@ static void enqueue_task(struct task_str */ static inline void enqueue_task_head(struct task_struct *p, prio_array_t *array) { - list_add(&p->run_list, array->queue + p->prio); - __set_bit(p->prio, array->bitmap); + list_add(&p->u.ingosched.run_list, array->queue + p->u.ingosched.prio); + __set_bit(p->u.ingosched.prio, array->bitmap); array->nr_active++; - p->array = array; + p->u.ingosched.array = array; +} + +static void ingo_set_oom_timeslice(task_t *p) +{ + p->u.ingosched.time_slice = HZ; } /* @@ -618,7 +612,7 @@ static int effective_prio(task_t *p) int bonus, prio; if (rt_task(p)) - return p->prio; + return p->u.ingosched.prio; bonus = CURRENT_BONUS(p) - MAX_BONUS / 2; @@ -650,7 +644,7 @@ static inline void __activate_idle_task( static void recalc_task_prio(task_t *p, unsigned long long now) { - unsigned long long __sleep_time = now - p->timestamp; + unsigned long long __sleep_time = now - p->u.ingosched.timestamp; unsigned long sleep_time; if (__sleep_time > NS_MAX_SLEEP_AVG) @@ -665,12 +659,12 @@ static void recalc_task_prio(task_t *p, * prevent them suddenly becoming cpu hogs and starving * other processes. */ - if (p->mm && p->activated != -1 && + if (p->mm && p->u.ingosched.activated != -1 && sleep_time > INTERACTIVE_SLEEP(p)) { - p->sleep_avg = JIFFIES_TO_NS(MAX_SLEEP_AVG - + p->u.ingosched.sleep_avg = JIFFIES_TO_NS(MAX_SLEEP_AVG - DEF_TIMESLICE); if (!HIGH_CREDIT(p)) - p->interactive_credit++; + p->u.ingosched.interactive_credit++; } else { /* * The lower the sleep avg a task has the more @@ -691,12 +685,12 @@ static void recalc_task_prio(task_t *p, * sleep are limited in their sleep_avg rise as they * are likely to be cpu hogs waiting on I/O */ - if (p->activated == -1 && !HIGH_CREDIT(p) && p->mm) { - if (p->sleep_avg >= INTERACTIVE_SLEEP(p)) + if (p->u.ingosched.activated == -1 && !HIGH_CREDIT(p) && p->mm) { + if (p->u.ingosched.sleep_avg >= INTERACTIVE_SLEEP(p)) sleep_time = 0; - else if (p->sleep_avg + sleep_time >= + else if (p->u.ingosched.sleep_avg + sleep_time >= INTERACTIVE_SLEEP(p)) { - p->sleep_avg = INTERACTIVE_SLEEP(p); + p->u.ingosched.sleep_avg = INTERACTIVE_SLEEP(p); sleep_time = 0; } } @@ -705,21 +699,21 @@ static void recalc_task_prio(task_t *p, * This code gives a bonus to interactive tasks. * * The boost works by updating the 'average sleep time' - * value here, based on ->timestamp. The more time a + * value here, based on ->u.ingosched.timestamp. The more time a * task spends sleeping, the higher the average gets - * and the higher the priority boost gets as well. */ - p->sleep_avg += sleep_time; + p->u.ingosched.sleep_avg += sleep_time; - if (p->sleep_avg > NS_MAX_SLEEP_AVG) { - p->sleep_avg = NS_MAX_SLEEP_AVG; + if (p->u.ingosched.sleep_avg > NS_MAX_SLEEP_AVG) { + p->u.ingosched.sleep_avg = NS_MAX_SLEEP_AVG; if (!HIGH_CREDIT(p)) - p->interactive_credit++; + p->u.ingosched.interactive_credit++; } } } - p->prio = effective_prio(p); + p->u.ingosched.prio = effective_prio(p); } /* @@ -748,7 +742,7 @@ static void activate_task(task_t *p, run * This checks to make sure it's not an uninterruptible task * that is now waking up. */ - if (!p->activated) { + if (!p->u.ingosched.activated) { /* * Tasks which were woken up by interrupts (ie. hw events) * are most likely of interactive nature. So we give them @@ -757,16 +751,16 @@ static void activate_task(task_t *p, run * on a CPU, first time around: */ if (in_interrupt()) - p->activated = 2; + p->u.ingosched.activated = 2; else { /* * Normal first-time wakeups get a credit too for * on-runqueue time, but it will be weighted down: */ - p->activated = 1; + p->u.ingosched.activated = 1; } } - p->timestamp = now; + p->u.ingosched.timestamp = now; __activate_task(p, rq); } @@ -779,8 +773,8 @@ static void deactivate_task(struct task_ rq->nr_running--; if (p->state == TASK_UNINTERRUPTIBLE) rq->nr_uninterruptible++; - dequeue_task(p, p->array); - p->array = NULL; + dequeue_task(p, p->u.ingosched.array); + p->u.ingosched.array = NULL; } /* @@ -816,7 +810,7 @@ static inline void resched_task(task_t * * task_curr - is this task currently executing on a CPU? * @p: the task in question. */ -inline int task_curr(const task_t *p) +static int ingo_task_curr(const task_t *p) { return cpu_curr(task_cpu(p)) == p; } @@ -853,7 +847,7 @@ static int migrate_task(task_t *p, int d * If the task is not on a runqueue (and not running), then * it is sufficient to simply update the task's cpu field. */ - if (!p->array && !task_running(rq, p)) { + if (!p->u.ingosched.array && !task_running(rq, p)) { set_task_cpu(p, dest_cpu); return 0; } @@ -875,7 +869,7 @@ static int migrate_task(task_t *p, int d * smp_call_function() if an IPI is sent by the same process we are * waiting to become inactive. */ -void wait_task_inactive(task_t * p) +static void ingo_wait_task_inactive(task_t * p) { unsigned long flags; runqueue_t *rq; @@ -884,7 +878,7 @@ void wait_task_inactive(task_t * p) repeat: rq = task_rq_lock(p, &flags); /* Must be off runqueue entirely, not preempted. */ - if (unlikely(p->array)) { + if (unlikely(p->u.ingosched.array)) { /* If it's preempted, we yield. It could be a while. */ preempted = !task_running(rq, p); task_rq_unlock(rq, &flags); @@ -896,24 +890,6 @@ repeat: task_rq_unlock(rq, &flags); } -/*** - * kick_process - kick a running thread to enter/exit the kernel - * @p: the to-be-kicked thread - * - * Cause a process which is running on another CPU to enter - * kernel-mode, without any delay. (to get signals handled.) - */ -void kick_process(task_t *p) -{ - int cpu; - - preempt_disable(); - cpu = task_cpu(p); - if ((cpu != smp_processor_id()) && task_curr(p)) - smp_send_reschedule(cpu); - preempt_enable(); -} - /* * Return a low guess at the load of a migration-source cpu. * @@ -993,7 +969,7 @@ static inline int wake_idle(int cpu, tas * * returns failure only if the task is already active. */ -static int try_to_wake_up(task_t * p, unsigned int state, int sync) +static int ingo_try_to_wake_up(task_t * p, unsigned int state, int sync) { int cpu, this_cpu, success = 0; unsigned long flags; @@ -1011,7 +987,7 @@ static int try_to_wake_up(task_t * p, un if (!(old_state & state)) goto out; - if (p->array) + if (p->u.ingosched.array) goto out_running; cpu = task_cpu(p); @@ -1090,7 +1066,7 @@ out_set_cpu: old_state = p->state; if (!(old_state & state)) goto out; - if (p->array) + if (p->u.ingosched.array) goto out_running; this_cpu = smp_processor_id(); @@ -1105,7 +1081,7 @@ out_activate: * Tasks on involuntary sleep don't earn * sleep_avg beyond just interactive state. */ - p->activated = -1; + p->u.ingosched.activated = -1; } /* @@ -1131,19 +1107,6 @@ out: return success; } -int fastcall wake_up_process(task_t * p) -{ - return try_to_wake_up(p, TASK_STOPPED | TASK_TRACED | - TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE, 0); -} - -EXPORT_SYMBOL(wake_up_process); - -int fastcall wake_up_state(task_t *p, unsigned int state) -{ - return try_to_wake_up(p, state, 0); -} - #ifdef CONFIG_SMP static int find_idlest_cpu(struct task_struct *p, int this_cpu, struct sched_domain *sd); @@ -1153,7 +1116,7 @@ static int find_idlest_cpu(struct task_s * Perform scheduler related setup for a newly forked process p. * p is forked by current. */ -void fastcall sched_fork(task_t *p) +static void ingo_sched_fork(task_t *p) { /* * We mark the process as running here, but have not actually @@ -1162,8 +1125,8 @@ void fastcall sched_fork(task_t *p) * event cannot wake it up and insert it on the runqueue either. */ p->state = TASK_RUNNING; - INIT_LIST_HEAD(&p->run_list); - p->array = NULL; + INIT_LIST_HEAD(&p->u.ingosched.run_list); + p->u.ingosched.array = NULL; spin_lock_init(&p->switch_lock); #ifdef CONFIG_SCHEDSTATS memset(&p->sched_info, 0, sizeof(p->sched_info)); @@ -1183,21 +1146,21 @@ void fastcall sched_fork(task_t *p) * resulting in more scheduling fairness. */ local_irq_disable(); - p->time_slice = (current->time_slice + 1) >> 1; + p->u.ingosched.time_slice = (current->u.ingosched.time_slice + 1) >> 1; /* * The remainder of the first timeslice might be recovered by * the parent if the child exits early enough. */ - p->first_time_slice = 1; - current->time_slice >>= 1; - p->timestamp = sched_clock(); - if (unlikely(!current->time_slice)) { + p->u.ingosched.first_time_slice = 1; + current->u.ingosched.time_slice >>= 1; + p->u.ingosched.timestamp = sched_clock(); + if (unlikely(!current->u.ingosched.time_slice)) { /* * This case is rare, it happens when the parent has only * a single jiffy left from its timeslice. Taking the * runqueue lock is not a problem. */ - current->time_slice = 1; + current->u.ingosched.time_slice = 1; preempt_disable(); scheduler_tick(); local_irq_enable(); @@ -1213,7 +1176,7 @@ void fastcall sched_fork(task_t *p) * that must be done for every newly created context, then puts the task * on the runqueue and wakes it. */ -void fastcall wake_up_new_task(task_t * p, unsigned long clone_flags) +static void ingo_wake_up_new_task(task_t * p, unsigned long clone_flags) { unsigned long flags; int this_cpu, cpu; @@ -1232,12 +1195,12 @@ void fastcall wake_up_new_task(task_t * * from forking tasks that are max-interactive. The parent * (current) is done further down, under its lock. */ - p->sleep_avg = JIFFIES_TO_NS(CURRENT_BONUS(p) * + p->u.ingosched.sleep_avg = JIFFIES_TO_NS(CURRENT_BONUS(p) * CHILD_PENALTY / 100 * MAX_SLEEP_AVG / MAX_BONUS); - p->interactive_credit = 0; + p->u.ingosched.interactive_credit = 0; - p->prio = effective_prio(p); + p->u.ingosched.prio = effective_prio(p); if (likely(cpu == this_cpu)) { if (!(clone_flags & CLONE_VM)) { @@ -1246,13 +1209,13 @@ void fastcall wake_up_new_task(task_t * * do child-runs-first in anticipation of an exec. This * usually avoids a lot of COW overhead. */ - if (unlikely(!current->array)) + if (unlikely(!current->u.ingosched.array)) __activate_task(p, rq); else { - p->prio = current->prio; - list_add_tail(&p->run_list, ¤t->run_list); - p->array = current->array; - p->array->nr_active++; + p->u.ingosched.prio = current->u.ingosched.prio; + list_add_tail(&p->u.ingosched.run_list, ¤t->u.ingosched.run_list); + p->u.ingosched.array = current->u.ingosched.array; + p->u.ingosched.array->nr_active++; rq->nr_running++; } set_need_resched(); @@ -1273,7 +1236,7 @@ void fastcall wake_up_new_task(task_t * * Not the local CPU - must adjust timestamp. This should * get optimised away in the !CONFIG_SMP case. */ - p->timestamp = (p->timestamp - this_rq->timestamp_last_tick) + p->u.ingosched.timestamp = (p->u.ingosched.timestamp - this_rq->timestamp_last_tick) + rq->timestamp_last_tick; __activate_task(p, rq); if (TASK_PREEMPTS_CURR(p, rq)) @@ -1282,12 +1245,12 @@ void fastcall wake_up_new_task(task_t * schedstat_inc(rq, wunt_moved); /* * Parent and child are on different CPUs, now get the - * parent runqueue to update the parent's ->sleep_avg: + * parent runqueue to update the parent's ->u.ingosched.sleep_avg: */ task_rq_unlock(rq, &flags); this_rq = task_rq_lock(current, &flags); } - current->sleep_avg = JIFFIES_TO_NS(CURRENT_BONUS(current) * + current->u.ingosched.sleep_avg = JIFFIES_TO_NS(CURRENT_BONUS(current) * PARENT_PENALTY / 100 * MAX_SLEEP_AVG / MAX_BONUS); task_rq_unlock(this_rq, &flags); } @@ -1301,7 +1264,7 @@ void fastcall wake_up_new_task(task_t * * artificially, because any timeslice recovered here * was given away by the parent in the first place.) */ -void fastcall sched_exit(task_t * p) +static void ingo_sched_exit(task_t * p) { unsigned long flags; runqueue_t *rq; @@ -1311,14 +1274,14 @@ void fastcall sched_exit(task_t * p) * the sleep_avg of the parent as well. */ rq = task_rq_lock(p->parent, &flags); - if (p->first_time_slice) { - p->parent->time_slice += p->time_slice; - if (unlikely(p->parent->time_slice > task_timeslice(p))) - p->parent->time_slice = task_timeslice(p); - } - if (p->sleep_avg < p->parent->sleep_avg) - p->parent->sleep_avg = p->parent->sleep_avg / - (EXIT_WEIGHT + 1) * EXIT_WEIGHT + p->sleep_avg / + if (p->u.ingosched.first_time_slice) { + p->parent->u.ingosched.time_slice += p->u.ingosched.time_slice; + if (unlikely(p->parent->u.ingosched.time_slice > task_timeslice(p))) + p->parent->u.ingosched.time_slice = task_timeslice(p); + } + if (p->u.ingosched.sleep_avg < p->parent->u.ingosched.sleep_avg) + p->parent->u.ingosched.sleep_avg = p->parent->u.ingosched.sleep_avg / + (EXIT_WEIGHT + 1) * EXIT_WEIGHT + p->u.ingosched.sleep_avg / (EXIT_WEIGHT + 1); task_rq_unlock(rq, &flags); } @@ -1367,7 +1330,7 @@ static void finish_task_switch(task_t *p * schedule_tail - first thing a freshly forked thread must call. * @prev: the thread we just switched away from. */ -asmlinkage void schedule_tail(task_t *prev) +static void ingo_schedule_tail(task_t *prev) { finish_task_switch(prev); @@ -1411,7 +1374,7 @@ task_t * context_switch(runqueue_t *rq, * threads, current number of uninterruptible-sleeping threads, total * number of context switches performed since bootup. */ -unsigned long nr_running(void) +static unsigned long ingo_nr_running(void) { unsigned long i, sum = 0; @@ -1421,7 +1384,7 @@ unsigned long nr_running(void) return sum; } -unsigned long nr_uninterruptible(void) +static unsigned long ingo_nr_uninterruptible(void) { unsigned long i, sum = 0; @@ -1431,7 +1394,7 @@ unsigned long nr_uninterruptible(void) return sum; } -unsigned long long nr_context_switches(void) +static unsigned long long ingo_nr_context_switches(void) { unsigned long long i, sum = 0; @@ -1441,7 +1404,7 @@ unsigned long long nr_context_switches(v return sum; } -unsigned long nr_iowait(void) +static unsigned long ingo_nr_iowait(void) { unsigned long i, sum = 0; @@ -1587,7 +1550,7 @@ out: * execve() is a valuable balancing opportunity, because at this point * the task has the smallest effective memory and cache footprint. */ -void sched_exec(void) +static void ingo_sched_exec(void) { struct sched_domain *tmp, *sd = NULL; int new_cpu, this_cpu = get_cpu(); @@ -1628,7 +1591,7 @@ void pull_task(runqueue_t *src_rq, prio_ set_task_cpu(p, this_cpu); this_rq->nr_running++; enqueue_task(p, this_array); - p->timestamp = (p->timestamp - src_rq->timestamp_last_tick) + p->u.ingosched.timestamp = (p->u.ingosched.timestamp - src_rq->timestamp_last_tick) + this_rq->timestamp_last_tick; /* * Note that idle threads have a prio of MAX_PRIO, for this test @@ -1724,7 +1687,7 @@ skip_bitmap: head = array->queue + idx; curr = head->prev; skip_queue: - tmp = list_entry(curr, task_t, run_list); + tmp = list_entry(curr, task_t, u.ingosched.run_list); curr = curr->prev; @@ -2230,10 +2193,6 @@ static inline int wake_priority_sleeper( return ret; } -DEFINE_PER_CPU(struct kernel_stat, kstat); - -EXPORT_PER_CPU_SYMBOL(kstat); - /* * We place interactive tasks back into the active array, if possible. * @@ -2320,7 +2279,7 @@ static void check_rlimit(struct task_str * @hardirq_offset: the offset to subtract from hardirq_count() * @cputime: the cpu time spent in user space since the last update */ -void account_user_time(struct task_struct *p, cputime_t cputime) +static void ingo_account_user_time(struct task_struct *p, cputime_t cputime) { struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; cputime64_t tmp; @@ -2347,7 +2306,7 @@ void account_user_time(struct task_struc * @hardirq_offset: the offset to subtract from hardirq_count() * @cputime: the cpu time spent in kernel space since the last update */ -void account_system_time(struct task_struct *p, int hardirq_offset, +static void ingo_account_system_time(struct task_struct *p, int hardirq_offset, cputime_t cputime) { struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; @@ -2380,7 +2339,7 @@ void account_system_time(struct task_str * @p: the process from which the cpu time has been stolen * @steal: the cpu time spent in involuntary wait */ -void account_steal_time(struct task_struct *p, cputime_t steal) +static void ingo_account_steal_time(struct task_struct *p, cputime_t steal) { struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; cputime64_t steal64 = cputime_to_cputime64(steal); @@ -2399,7 +2358,7 @@ void account_steal_time(struct task_stru * It also gets called by the fork code, when changing the parent's * timeslices. */ -void scheduler_tick(void) +static void ingo_scheduler_tick(void) { int cpu = smp_processor_id(); runqueue_t *rq = this_rq(); @@ -2415,7 +2374,7 @@ void scheduler_tick(void) } /* Task might have expired already, but not scheduled off yet */ - if (p->array != rq->active) { + if (p->u.ingosched.array != rq->active) { set_tsk_need_resched(p); goto out; } @@ -2432,9 +2391,9 @@ void scheduler_tick(void) * RR tasks need a special form of timeslice management. * FIFO tasks have no timeslices. */ - if ((p->policy == SCHED_RR) && !--p->time_slice) { - p->time_slice = task_timeslice(p); - p->first_time_slice = 0; + if ((p->policy == SCHED_RR) && !--p->u.ingosched.time_slice) { + p->u.ingosched.time_slice = task_timeslice(p); + p->u.ingosched.first_time_slice = 0; set_tsk_need_resched(p); /* put it at the end of the queue: */ @@ -2443,12 +2402,12 @@ void scheduler_tick(void) } goto out_unlock; } - if (!--p->time_slice) { + if (!--p->u.ingosched.time_slice) { dequeue_task(p, rq->active); set_tsk_need_resched(p); - p->prio = effective_prio(p); - p->time_slice = task_timeslice(p); - p->first_time_slice = 0; + p->u.ingosched.prio = effective_prio(p); + p->u.ingosched.time_slice = task_timeslice(p); + p->u.ingosched.first_time_slice = 0; if (!rq->expired_timestamp) rq->expired_timestamp = jiffies; @@ -2476,13 +2435,13 @@ void scheduler_tick(void) * delta range with at least TIMESLICE_GRANULARITY to requeue. */ if (TASK_INTERACTIVE(p) && !((task_timeslice(p) - - p->time_slice) % TIMESLICE_GRANULARITY(p)) && - (p->time_slice >= TIMESLICE_GRANULARITY(p)) && - (p->array == rq->active)) { + p->u.ingosched.time_slice) % TIMESLICE_GRANULARITY(p)) && + (p->u.ingosched.time_slice >= TIMESLICE_GRANULARITY(p)) && + (p->u.ingosched.array == rq->active)) { dequeue_task(p, rq->active); set_tsk_need_resched(p); - p->prio = effective_prio(p); + p->u.ingosched.prio = effective_prio(p); enqueue_task(p, rq->active); } } @@ -2571,7 +2530,7 @@ static inline int dependent_sleeper(int BUG_ON(!array->nr_active); p = list_entry(array->queue[sched_find_first_bit(array->bitmap)].next, - task_t, run_list); + task_t, u.ingosched.run_list); for_each_cpu_mask(i, sibling_map) { runqueue_t *smt_rq = cpu_rq(i); @@ -2585,7 +2544,7 @@ static inline int dependent_sleeper(int * task from using an unfair proportion of the * physical cpu's resources. -ck */ - if (((smt_curr->time_slice * (100 - sd->per_cpu_gain) / 100) > + if (((smt_curr->u.ingosched.time_slice * (100 - sd->per_cpu_gain) / 100) > task_timeslice(p) || rt_task(smt_curr)) && p->mm && smt_curr->mm && !rt_task(p)) ret = 1; @@ -2595,7 +2554,7 @@ static inline int dependent_sleeper(int * or wake it up if it has been put to sleep for priority * reasons. */ - if ((((p->time_slice * (100 - sd->per_cpu_gain) / 100) > + if ((((p->u.ingosched.time_slice * (100 - sd->per_cpu_gain) / 100) > task_timeslice(smt_curr) || rt_task(p)) && smt_curr->mm && p->mm && !rt_task(smt_curr)) || (smt_curr == smt_rq->idle && smt_rq->nr_running)) @@ -2617,42 +2576,10 @@ static inline int dependent_sleeper(int } #endif -#if defined(CONFIG_PREEMPT) && defined(CONFIG_DEBUG_PREEMPT) - -void fastcall add_preempt_count(int val) -{ - /* - * Underflow? - */ - BUG_ON(((int)preempt_count() < 0)); - preempt_count() += val; - /* - * Spinlock count overflowing soon? - */ - BUG_ON((preempt_count() & PREEMPT_MASK) >= PREEMPT_MASK-10); -} -EXPORT_SYMBOL(add_preempt_count); - -void fastcall sub_preempt_count(int val) -{ - /* - * Underflow? - */ - BUG_ON(val > preempt_count()); - /* - * Is the spinlock portion underflowing? - */ - BUG_ON((val < PREEMPT_MASK) && !(preempt_count() & PREEMPT_MASK)); - preempt_count() -= val; -} -EXPORT_SYMBOL(sub_preempt_count); - -#endif - /* * schedule() is the main scheduler function. */ -asmlinkage void __sched schedule(void) +static void __sched ingo_schedule(void) { long *switch_count; task_t *prev, *next; @@ -2696,8 +2623,8 @@ need_resched_nonpreemptible: schedstat_inc(rq, sched_cnt); now = sched_clock(); - if (likely(now - prev->timestamp < NS_MAX_SLEEP_AVG)) - run_time = now - prev->timestamp; + if (likely(now - prev->u.ingosched.timestamp < NS_MAX_SLEEP_AVG)) + run_time = now - prev->u.ingosched.timestamp; else run_time = NS_MAX_SLEEP_AVG; @@ -2773,20 +2700,20 @@ go_idle: idx = sched_find_first_bit(array->bitmap); queue = array->queue + idx; - next = list_entry(queue->next, task_t, run_list); + next = list_entry(queue->next, task_t, u.ingosched.run_list); - if (!rt_task(next) && next->activated > 0) { - unsigned long long delta = now - next->timestamp; + if (!rt_task(next) && next->u.ingosched.activated > 0) { + unsigned long long delta = now - next->u.ingosched.timestamp; - if (next->activated == 1) + if (next->u.ingosched.activated == 1) delta = delta * (ON_RUNQUEUE_WEIGHT * 128 / 100) / 128; - array = next->array; + array = next->u.ingosched.array; dequeue_task(next, array); - recalc_task_prio(next, next->timestamp + delta); + recalc_task_prio(next, next->u.ingosched.timestamp + delta); enqueue_task(next, array); } - next->activated = 0; + next->u.ingosched.activated = 0; switch_tasks: if (next == rq->idle) schedstat_inc(rq, sched_goidle); @@ -2794,17 +2721,17 @@ switch_tasks: clear_tsk_need_resched(prev); rcu_qsctr_inc(task_cpu(prev)); - prev->sleep_avg -= run_time; - if ((long)prev->sleep_avg <= 0) { - prev->sleep_avg = 0; + prev->u.ingosched.sleep_avg -= run_time; + if ((long)prev->u.ingosched.sleep_avg <= 0) { + prev->u.ingosched.sleep_avg = 0; if (!(HIGH_CREDIT(prev) || LOW_CREDIT(prev))) - prev->interactive_credit--; + prev->u.ingosched.interactive_credit--; } - prev->timestamp = prev->last_ran = now; + prev->u.ingosched.timestamp = prev->u.ingosched.last_ran = now; sched_info_switch(prev, next); if (likely(prev != next)) { - next->timestamp = now; + next->u.ingosched.timestamp = now; rq->nr_switches++; rq->curr = next; ++*switch_count; @@ -2825,169 +2752,7 @@ switch_tasks: goto need_resched; } -EXPORT_SYMBOL(schedule); - -#ifdef CONFIG_PREEMPT -/* - * this is is the entry point to schedule() from in-kernel preemption - * off of preempt_enable. Kernel preemptions off return from interrupt - * occur there and call schedule directly. - */ -asmlinkage void __sched preempt_schedule(void) -{ - struct thread_info *ti = current_thread_info(); -#ifdef CONFIG_PREEMPT_BKL - struct task_struct *task = current; - int saved_lock_depth; -#endif - /* - * If there is a non-zero preempt_count or interrupts are disabled, - * we do not want to preempt the current task. Just return.. - */ - if (unlikely(ti->preempt_count || irqs_disabled())) - return; - -need_resched: - add_preempt_count(PREEMPT_ACTIVE); - /* - * We keep the big kernel semaphore locked, but we - * clear ->lock_depth so that schedule() doesnt - * auto-release the semaphore: - */ -#ifdef CONFIG_PREEMPT_BKL - saved_lock_depth = task->lock_depth; - task->lock_depth = -1; -#endif - schedule(); -#ifdef CONFIG_PREEMPT_BKL - task->lock_depth = saved_lock_depth; -#endif - sub_preempt_count(PREEMPT_ACTIVE); - - /* we could miss a preemption opportunity between schedule and now */ - barrier(); - if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) - goto need_resched; -} - -EXPORT_SYMBOL(preempt_schedule); -#endif /* CONFIG_PREEMPT */ - -int default_wake_function(wait_queue_t *curr, unsigned mode, int sync, void *key) -{ - task_t *p = curr->task; - return try_to_wake_up(p, mode, sync); -} - -EXPORT_SYMBOL(default_wake_function); - -/* - * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just - * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve - * number) then we wake all the non-exclusive tasks and one exclusive task. - * - * There are circumstances in which we can try to wake a task which has already - * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns - * zero in this (rare) case, and we handle it by continuing to scan the queue. - */ -static void __wake_up_common(wait_queue_head_t *q, unsigned int mode, - int nr_exclusive, int sync, void *key) -{ - struct list_head *tmp, *next; - - list_for_each_safe(tmp, next, &q->task_list) { - wait_queue_t *curr; - unsigned flags; - curr = list_entry(tmp, wait_queue_t, task_list); - flags = curr->flags; - if (curr->func(curr, mode, sync, key) && - (flags & WQ_FLAG_EXCLUSIVE) && - !--nr_exclusive) - break; - } -} - -/** - * __wake_up - wake up threads blocked on a waitqueue. - * @q: the waitqueue - * @mode: which threads - * @nr_exclusive: how many wake-one or wake-many threads to wake up - */ -void fastcall __wake_up(wait_queue_head_t *q, unsigned int mode, - int nr_exclusive, void *key) -{ - unsigned long flags; - - spin_lock_irqsave(&q->lock, flags); - __wake_up_common(q, mode, nr_exclusive, 0, key); - spin_unlock_irqrestore(&q->lock, flags); -} - -EXPORT_SYMBOL(__wake_up); - -/* - * Same as __wake_up but called with the spinlock in wait_queue_head_t held. - */ -void fastcall __wake_up_locked(wait_queue_head_t *q, unsigned int mode) -{ - __wake_up_common(q, mode, 1, 0, NULL); -} - -/** - * __wake_up - sync- wake up threads blocked on a waitqueue. - * @q: the waitqueue - * @mode: which threads - * @nr_exclusive: how many wake-one or wake-many threads to wake up - * - * The sync wakeup differs that the waker knows that it will schedule - * away soon, so while the target thread will be woken up, it will not - * be migrated to another CPU - ie. the two threads are 'synchronized' - * with each other. This can prevent needless bouncing between CPUs. - * - * On UP it can prevent extra preemption. - */ -void fastcall __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive) -{ - unsigned long flags; - int sync = 1; - - if (unlikely(!q)) - return; - - if (unlikely(!nr_exclusive)) - sync = 0; - - spin_lock_irqsave(&q->lock, flags); - __wake_up_common(q, mode, nr_exclusive, sync, NULL); - spin_unlock_irqrestore(&q->lock, flags); -} -EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */ - -void fastcall complete(struct completion *x) -{ - unsigned long flags; - - spin_lock_irqsave(&x->wait.lock, flags); - x->done++; - __wake_up_common(&x->wait, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, - 1, 0, NULL); - spin_unlock_irqrestore(&x->wait.lock, flags); -} -EXPORT_SYMBOL(complete); - -void fastcall complete_all(struct completion *x) -{ - unsigned long flags; - - spin_lock_irqsave(&x->wait.lock, flags); - x->done += UINT_MAX/2; - __wake_up_common(&x->wait, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, - 0, 0, NULL); - spin_unlock_irqrestore(&x->wait.lock, flags); -} -EXPORT_SYMBOL(complete_all); - -void fastcall __sched wait_for_completion(struct completion *x) +static void __sched ingo_wait_for_completion(struct completion *x) { might_sleep(); spin_lock_irq(&x->wait.lock); @@ -3009,78 +2774,7 @@ void fastcall __sched wait_for_completio } EXPORT_SYMBOL(wait_for_completion); -#define SLEEP_ON_VAR \ - unsigned long flags; \ - wait_queue_t wait; \ - init_waitqueue_entry(&wait, current); - -#define SLEEP_ON_HEAD \ - spin_lock_irqsave(&q->lock,flags); \ - __add_wait_queue(q, &wait); \ - spin_unlock(&q->lock); - -#define SLEEP_ON_TAIL \ - spin_lock_irq(&q->lock); \ - __remove_wait_queue(q, &wait); \ - spin_unlock_irqrestore(&q->lock, flags); - -void fastcall __sched interruptible_sleep_on(wait_queue_head_t *q) -{ - SLEEP_ON_VAR - - current->state = TASK_INTERRUPTIBLE; - - SLEEP_ON_HEAD - schedule(); - SLEEP_ON_TAIL -} - -EXPORT_SYMBOL(interruptible_sleep_on); - -long fastcall __sched interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout) -{ - SLEEP_ON_VAR - - current->state = TASK_INTERRUPTIBLE; - - SLEEP_ON_HEAD - timeout = schedule_timeout(timeout); - SLEEP_ON_TAIL - - return timeout; -} - -EXPORT_SYMBOL(interruptible_sleep_on_timeout); - -void fastcall __sched sleep_on(wait_queue_head_t *q) -{ - SLEEP_ON_VAR - - current->state = TASK_UNINTERRUPTIBLE; - - SLEEP_ON_HEAD - schedule(); - SLEEP_ON_TAIL -} - -EXPORT_SYMBOL(sleep_on); - -long fastcall __sched sleep_on_timeout(wait_queue_head_t *q, long timeout) -{ - SLEEP_ON_VAR - - current->state = TASK_UNINTERRUPTIBLE; - - SLEEP_ON_HEAD - timeout = schedule_timeout(timeout); - SLEEP_ON_TAIL - - return timeout; -} - -EXPORT_SYMBOL(sleep_on_timeout); - -void set_user_nice(task_t *p, long nice) +static void ingo_set_user_nice(task_t *p, long nice) { unsigned long flags; prio_array_t *array; @@ -3104,15 +2798,15 @@ void set_user_nice(task_t *p, long nice) p->static_prio = NICE_TO_PRIO(nice); goto out_unlock; } - array = p->array; + array = p->u.ingosched.array; if (array) dequeue_task(p, array); - old_prio = p->prio; + old_prio = p->u.ingosched.prio; new_prio = NICE_TO_PRIO(nice); delta = new_prio - old_prio; p->static_prio = NICE_TO_PRIO(nice); - p->prio += delta; + p->u.ingosched.prio += delta; if (array) { enqueue_task(p, array); @@ -3127,8 +2821,6 @@ out_unlock: task_rq_unlock(rq, &flags); } -EXPORT_SYMBOL(set_user_nice); - #ifdef CONFIG_KGDB struct task_struct *kgdb_get_idle(int this_cpu) { @@ -3136,50 +2828,6 @@ struct task_struct *kgdb_get_idle(int th } #endif -#ifdef __ARCH_WANT_SYS_NICE - -/* - * sys_nice - change the priority of the current process. - * @increment: priority increment - * - * sys_setpriority is a more generic, but much slower function that - * does similar things. - */ -asmlinkage long sys_nice(int increment) -{ - int retval; - long nice; - - /* - * Setpriority might change our priority at the same moment. - * We don't have to worry. Conceptually one call occurs first - * and we have a single winner. - */ - if (increment < 0) { - if (!capable(CAP_SYS_NICE)) - return -EPERM; - if (increment < -40) - increment = -40; - } - if (increment > 40) - increment = 40; - - nice = PRIO_TO_NICE(current->static_prio) + increment; - if (nice < -20) - nice = -20; - if (nice > 19) - nice = 19; - - retval = security_task_setnice(current, nice); - if (retval) - return retval; - - set_user_nice(current, nice); - return 0; -} - -#endif - /** * task_prio - return the priority value of a given task. * @p: the task in question. @@ -3188,33 +2836,29 @@ asmlinkage long sys_nice(int increment) * RT tasks are offset by -200. Normal tasks are centered * around 0, value goes from -16 to +15. */ -int task_prio(const task_t *p) +static int ingo_task_prio(const task_t *p) { - return p->prio - MAX_RT_PRIO; + return p->u.ingosched.prio - MAX_RT_PRIO; } /** * task_nice - return the nice value of a given task. * @p: the task in question. */ -int task_nice(const task_t *p) +static int ingo_task_nice(const task_t *p) { return TASK_NICE(p); } -EXPORT_SYMBOL(task_nice); - /** * idle_cpu - is a given cpu idle currently? * @cpu: the processor in question. */ -int idle_cpu(int cpu) +static int ingo_idle_cpu(int cpu) { return cpu_curr(cpu) == cpu_rq(cpu)->idle; } -EXPORT_SYMBOL_GPL(idle_cpu); - /** * find_process_by_pid - find a process with a matching PID value. * @pid: the pid in question. @@ -3227,19 +2871,19 @@ static inline task_t *find_process_by_pi /* Actually do priority change: must hold rq lock. */ static void __setscheduler(struct task_struct *p, int policy, int prio) { - BUG_ON(p->array); + BUG_ON(p->u.ingosched.array); p->policy = policy; p->rt_priority = prio; if (policy != SCHED_NORMAL) - p->prio = MAX_USER_RT_PRIO-1 - p->rt_priority; + p->u.ingosched.prio = MAX_USER_RT_PRIO-1 - p->rt_priority; else - p->prio = p->static_prio; + p->u.ingosched.prio = p->static_prio; } /* * setscheduler - change the scheduling policy and/or RT priority of a thread. */ -static int setscheduler(pid_t pid, int policy, struct sched_param __user *param) +static int ingo_setscheduler(pid_t pid, int policy, struct sched_param __user *param) { struct sched_param lp; int retval = -EINVAL; @@ -3308,11 +2952,11 @@ recheck: task_rq_unlock(rq, &flags); goto recheck; } - array = p->array; + array = p->u.ingosched.array; if (array) deactivate_task(p, task_rq(p)); retval = 0; - oldprio = p->prio; + oldprio = p->u.ingosched.prio; __setscheduler(p, policy, lp.sched_priority); if (array) { __activate_task(p, task_rq(p)); @@ -3322,7 +2966,7 @@ recheck: * this runqueue and our priority is higher than the current's */ if (task_running(rq, p)) { - if (p->prio > oldprio) + if (p->u.ingosched.prio > oldprio) resched_task(rq->curr); } else if (TASK_PREEMPTS_CURR(p, rq)) resched_task(rq->curr); @@ -3335,238 +2979,16 @@ out_nounlock: } /** - * sys_sched_setscheduler - set/change the scheduler policy and RT priority - * @pid: the pid in question. - * @policy: new policy - * @param: structure containing the new RT priority. - */ -asmlinkage long sys_sched_setscheduler(pid_t pid, int policy, - struct sched_param __user *param) -{ - return setscheduler(pid, policy, param); -} - -/** - * sys_sched_setparam - set/change the RT priority of a thread - * @pid: the pid in question. - * @param: structure containing the new RT priority. - */ -asmlinkage long sys_sched_setparam(pid_t pid, struct sched_param __user *param) -{ - return setscheduler(pid, -1, param); -} - -/** - * sys_sched_getscheduler - get the policy (scheduling class) of a thread - * @pid: the pid in question. - */ -asmlinkage long sys_sched_getscheduler(pid_t pid) -{ - int retval = -EINVAL; - task_t *p; - - if (pid < 0) - goto out_nounlock; - - retval = -ESRCH; - read_lock(&tasklist_lock); - p = find_process_by_pid(pid); - if (p) { - retval = security_task_getscheduler(p); - if (!retval) - retval = p->policy; - } - read_unlock(&tasklist_lock); - -out_nounlock: - return retval; -} - -/** - * sys_sched_getscheduler - get the RT priority of a thread - * @pid: the pid in question. - * @param: structure containing the RT priority. - */ -asmlinkage long sys_sched_getparam(pid_t pid, struct sched_param __user *param) -{ - struct sched_param lp; - int retval = -EINVAL; - task_t *p; - - if (!param || pid < 0) - goto out_nounlock; - - read_lock(&tasklist_lock); - p = find_process_by_pid(pid); - retval = -ESRCH; - if (!p) - goto out_unlock; - - retval = security_task_getscheduler(p); - if (retval) - goto out_unlock; - - lp.sched_priority = p->rt_priority; - read_unlock(&tasklist_lock); - - /* - * This one might sleep, we cannot do it with a spinlock held ... - */ - retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0; - -out_nounlock: - return retval; - -out_unlock: - read_unlock(&tasklist_lock); - return retval; -} - -long sched_setaffinity(pid_t pid, cpumask_t new_mask) -{ - task_t *p; - int retval; - cpumask_t cpus_allowed; - - lock_cpu_hotplug(); - read_lock(&tasklist_lock); - - p = find_process_by_pid(pid); - if (!p) { - read_unlock(&tasklist_lock); - unlock_cpu_hotplug(); - return -ESRCH; - } - - /* - * It is not safe to call set_cpus_allowed with the - * tasklist_lock held. We will bump the task_struct's - * usage count and then drop tasklist_lock. - */ - get_task_struct(p); - read_unlock(&tasklist_lock); - - retval = -EPERM; - if ((current->euid != p->euid) && (current->euid != p->uid) && - !capable(CAP_SYS_NICE)) - goto out_unlock; - - cpus_allowed = cpuset_cpus_allowed(p); - cpus_and(new_mask, new_mask, cpus_allowed); - retval = set_cpus_allowed(p, new_mask); - -out_unlock: - put_task_struct(p); - unlock_cpu_hotplug(); - return retval; -} - -static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, - cpumask_t *new_mask) -{ - if (len < sizeof(cpumask_t)) { - memset(new_mask, 0, sizeof(cpumask_t)); - } else if (len > sizeof(cpumask_t)) { - len = sizeof(cpumask_t); - } - return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0; -} - -/** - * sys_sched_setaffinity - set the cpu affinity of a process - * @pid: pid of the process - * @len: length in bytes of the bitmask pointed to by user_mask_ptr - * @user_mask_ptr: user-space pointer to the new cpu mask - */ -asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len, - unsigned long __user *user_mask_ptr) -{ - cpumask_t new_mask; - int retval; - - retval = get_user_cpu_mask(user_mask_ptr, len, &new_mask); - if (retval) - return retval; - - return sched_setaffinity(pid, new_mask); -} - -/* - * Represents all cpu's present in the system - * In systems capable of hotplug, this map could dynamically grow - * as new cpu's are detected in the system via any platform specific - * method, such as ACPI for e.g. - */ - -cpumask_t cpu_present_map; -EXPORT_SYMBOL(cpu_present_map); - -#ifndef CONFIG_SMP -cpumask_t cpu_online_map = CPU_MASK_ALL; -cpumask_t cpu_possible_map = CPU_MASK_ALL; -#endif - -long sched_getaffinity(pid_t pid, cpumask_t *mask) -{ - int retval; - task_t *p; - - lock_cpu_hotplug(); - read_lock(&tasklist_lock); - - retval = -ESRCH; - p = find_process_by_pid(pid); - if (!p) - goto out_unlock; - - retval = 0; - cpus_and(*mask, p->cpus_allowed, cpu_possible_map); - -out_unlock: - read_unlock(&tasklist_lock); - unlock_cpu_hotplug(); - if (retval) - return retval; - - return 0; -} - -/** - * sys_sched_getaffinity - get the cpu affinity of a process - * @pid: pid of the process - * @len: length in bytes of the bitmask pointed to by user_mask_ptr - * @user_mask_ptr: user-space pointer to hold the current cpu mask - */ -asmlinkage long sys_sched_getaffinity(pid_t pid, unsigned int len, - unsigned long __user *user_mask_ptr) -{ - int ret; - cpumask_t mask; - - if (len < sizeof(cpumask_t)) - return -EINVAL; - - ret = sched_getaffinity(pid, &mask); - if (ret < 0) - return ret; - - if (copy_to_user(user_mask_ptr, &mask, sizeof(cpumask_t))) - return -EFAULT; - - return sizeof(cpumask_t); -} - -/** * sys_sched_yield - yield the current processor to other threads. * * this function yields the current CPU by moving the calling thread * to the expired array. If there are no other threads running on this * CPU then this function will return. */ -asmlinkage long sys_sched_yield(void) +static long ingo_sys_sched_yield(void) { runqueue_t *rq = this_rq_lock(); - prio_array_t *array = current->array; + prio_array_t *array = current->u.ingosched.array; prio_array_t *target = rq->expired; schedstat_inc(rq, yld_cnt); @@ -3580,7 +3002,7 @@ asmlinkage long sys_sched_yield(void) if (rt_task(current)) target = rq->active; - if (current->array->nr_active == 1) { + if (current->u.ingosched.array->nr_active == 1) { schedstat_inc(rq, yld_act_empty); if (!rq->expired->nr_active) schedstat_inc(rq, yld_both_empty); @@ -3602,86 +3024,6 @@ asmlinkage long sys_sched_yield(void) return 0; } -static inline void __cond_resched(void) -{ - do { - add_preempt_count(PREEMPT_ACTIVE); - schedule(); - sub_preempt_count(PREEMPT_ACTIVE); - } while (need_resched()); -} - -int __sched cond_resched(void) -{ - if (need_resched()) { - __cond_resched(); - return 1; - } - return 0; -} - -EXPORT_SYMBOL(cond_resched); - -/* - * cond_resched_lock() - if a reschedule is pending, drop the given lock, - * call schedule, and on return reacquire the lock. - * - * This works OK both with and without CONFIG_PREEMPT. We do strange low-level - * operations here to prevent schedule() from being called twice (once via - * spin_unlock(), once by hand). - */ -int cond_resched_lock(spinlock_t * lock) -{ -#if defined(CONFIG_SMP) && defined(CONFIG_PREEMPT) - if (lock->break_lock) { - lock->break_lock = 0; - spin_unlock(lock); - cpu_relax(); - spin_lock(lock); - } -#endif - if (need_resched()) { - _raw_spin_unlock(lock); - preempt_enable_no_resched(); - __cond_resched(); - spin_lock(lock); - return 1; - } - return 0; -} - -EXPORT_SYMBOL(cond_resched_lock); - -int __sched cond_resched_softirq(void) -{ - BUG_ON(!in_softirq()); - - if (need_resched()) { - __local_bh_enable(); - __cond_resched(); - local_bh_disable(); - return 1; - } - return 0; -} - -EXPORT_SYMBOL(cond_resched_softirq); - - -/** - * yield - yield the current processor to other threads. - * - * this is a shortcut for kernel-space yielding - it marks the - * thread runnable and calls sys_sched_yield(). - */ -void __sched yield(void) -{ - set_current_state(TASK_RUNNING); - sys_sched_yield(); -} - -EXPORT_SYMBOL(yield); - /* * This task is about to go to sleep on IO. Increment rq->nr_iowait so * that process accounting knows that this is a task in IO wait state. @@ -3689,7 +3031,7 @@ EXPORT_SYMBOL(yield); * But don't do that if it is a deliberate, throttling IO wait (this task * has set its backing_dev_info: the queue against which it should throttle) */ -void __sched io_schedule(void) +static void __sched ingo_io_schedule(void) { struct runqueue *rq = &per_cpu(runqueues, _smp_processor_id()); @@ -3698,9 +3040,7 @@ void __sched io_schedule(void) atomic_dec(&rq->nr_iowait); } -EXPORT_SYMBOL(io_schedule); - -long __sched io_schedule_timeout(long timeout) +static long __sched ingo_io_schedule_timeout(long timeout) { struct runqueue *rq = &per_cpu(runqueues, _smp_processor_id()); long ret; @@ -3712,51 +3052,6 @@ long __sched io_schedule_timeout(long ti } /** - * sys_sched_get_priority_max - return maximum RT priority. - * @policy: scheduling class. - * - * this syscall returns the maximum rt_priority that can be used - * by a given scheduling class. - */ -asmlinkage long sys_sched_get_priority_max(int policy) -{ - int ret = -EINVAL; - - switch (policy) { - case SCHED_FIFO: - case SCHED_RR: - ret = MAX_USER_RT_PRIO-1; - break; - case SCHED_NORMAL: - ret = 0; - break; - } - return ret; -} - -/** - * sys_sched_get_priority_min - return minimum RT priority. - * @policy: scheduling class. - * - * this syscall returns the minimum rt_priority that can be used - * by a given scheduling class. - */ -asmlinkage long sys_sched_get_priority_min(int policy) -{ - int ret = -EINVAL; - - switch (policy) { - case SCHED_FIFO: - case SCHED_RR: - ret = 1; - break; - case SCHED_NORMAL: - ret = 0; - } - return ret; -} - -/** * sys_sched_rr_get_interval - return the default timeslice of a process. * @pid: pid of the process. * @interval: userspace pointer to the timeslice value. @@ -3764,8 +3059,8 @@ asmlinkage long sys_sched_get_priority_m * this syscall writes the default timeslice value of a given process * into the user-space timespec buffer. A value of '0' means infinity. */ -asmlinkage -long sys_sched_rr_get_interval(pid_t pid, struct timespec __user *interval) +static long +ingo_sys_sched_rr_get_interval(pid_t pid, struct timespec __user *interval) { int retval = -EINVAL; struct timespec t; @@ -3795,113 +3090,15 @@ out_unlock: return retval; } -static inline struct task_struct *eldest_child(struct task_struct *p) -{ - if (list_empty(&p->children)) return NULL; - return list_entry(p->children.next,struct task_struct,sibling); -} - -static inline struct task_struct *older_sibling(struct task_struct *p) -{ - if (p->sibling.prev==&p->parent->children) return NULL; - return list_entry(p->sibling.prev,struct task_struct,sibling); -} - -static inline struct task_struct *younger_sibling(struct task_struct *p) -{ - if (p->sibling.next==&p->parent->children) return NULL; - return list_entry(p->sibling.next,struct task_struct,sibling); -} - -static void show_task(task_t * p) -{ - task_t *relative; - unsigned state; - unsigned long free = 0; - static const char *stat_nam[] = { "R", "S", "D", "T", "t", "Z", "X" }; - - printk("%-13.13s ", p->comm); - state = p->state ? __ffs(p->state) + 1 : 0; - if (state < ARRAY_SIZE(stat_nam)) - printk(stat_nam[state]); - else - printk("?"); -#if (BITS_PER_LONG == 32) - if (state == TASK_RUNNING) - printk(" running "); - else - printk(" %08lX ", thread_saved_pc(p)); -#else - if (state == TASK_RUNNING) - printk(" running task "); - else - printk(" %016lx ", thread_saved_pc(p)); -#endif -#ifdef CONFIG_DEBUG_STACK_USAGE - { - unsigned long * n = (unsigned long *) (p->thread_info+1); - while (!*n) - n++; - free = (unsigned long) n - (unsigned long)(p->thread_info+1); - } -#endif - printk("%5lu %5d %6d ", free, p->pid, p->parent->pid); - if ((relative = eldest_child(p))) - printk("%5d ", relative->pid); - else - printk(" "); - if ((relative = younger_sibling(p))) - printk("%7d", relative->pid); - else - printk(" "); - if ((relative = older_sibling(p))) - printk(" %5d", relative->pid); - else - printk(" "); - if (!p->mm) - printk(" (L-TLB)\n"); - else - printk(" (NOTLB)\n"); - - if (state != TASK_RUNNING) - show_stack(p, NULL); -} - -void show_state(void) -{ - task_t *g, *p; - -#if (BITS_PER_LONG == 32) - printk("\n" - " sibling\n"); - printk(" task PC pid father child younger older\n"); -#else - printk("\n" - " sibling\n"); - printk(" task PC pid father child younger older\n"); -#endif - read_lock(&tasklist_lock); - do_each_thread(g, p) { - /* - * reset the NMI-timeout, listing all files on a slow - * console might take alot of time: - */ - touch_nmi_watchdog(); - show_task(p); - } while_each_thread(g, p); - - read_unlock(&tasklist_lock); -} - -void __devinit init_idle(task_t *idle, int cpu) +static void __devinit ingo_init_idle(task_t *idle, int cpu) { runqueue_t *rq = cpu_rq(cpu); unsigned long flags; - idle->sleep_avg = 0; - idle->interactive_credit = 0; - idle->array = NULL; - idle->prio = MAX_PRIO; + idle->u.ingosched.sleep_avg = 0; + idle->u.ingosched.interactive_credit = 0; + idle->u.ingosched.array = NULL; + idle->u.ingosched.prio = MAX_PRIO; idle->state = TASK_RUNNING; set_task_cpu(idle, cpu); @@ -3918,15 +3115,6 @@ void __devinit init_idle(task_t *idle, i #endif } -/* - * In a system that switches off the HZ timer nohz_cpu_mask - * indicates which cpus entered this state. This is used - * in the rcu update to wait only for active cpus. For system - * which do not switch off the HZ timer nohz_cpu_mask should - * always be CPU_MASK_NONE. - */ -cpumask_t nohz_cpu_mask = CPU_MASK_NONE; - #ifdef CONFIG_SMP /* * This is how migration works: @@ -3953,7 +3141,7 @@ cpumask_t nohz_cpu_mask = CPU_MASK_NONE; * task must not exit() & deallocate itself prematurely. The * call is not atomic; no spinlocks may be held. */ -int set_cpus_allowed(task_t *p, cpumask_t new_mask) +static int ingo_set_cpus_allowed(task_t *p, cpumask_t new_mask) { unsigned long flags; int ret = 0; @@ -3986,8 +3174,6 @@ out: return ret; } -EXPORT_SYMBOL_GPL(set_cpus_allowed); - /* * Move (not current) task off this cpu, onto dest cpu. We're doing * this because either it can't run here any more (set_cpus_allowed() @@ -4016,15 +3202,16 @@ static void __migrate_task(struct task_s goto out; set_task_cpu(p, dest_cpu); - if (p->array) { + if (p->u.ingosched.array) { /* * Sync timestamp with rq_dest's before activating. * The same thing could be achieved by doing this step * afterwards, and pretending it was a local activate. * This way is cleaner and logically correct. */ - p->timestamp = p->timestamp - rq_src->timestamp_last_tick - + rq_dest->timestamp_last_tick; + p->u.ingosched.timestamp = p->u.ingosched.timestamp - + rq_src->timestamp_last_tick + + rq_dest->timestamp_last_tick; deactivate_task(p, rq_src); activate_task(p, rq_dest, 0); if (TASK_PREEMPTS_CURR(p, rq_dest)) @@ -4164,7 +3351,7 @@ static void migrate_live_tasks(int src_c * It does so by boosting its priority to highest possible and adding it to * the _front_ of runqueue. Used by CPU offline code. */ -void sched_idle_next(void) +static void ingo_sched_idle_next(void) { int cpu = smp_processor_id(); runqueue_t *rq = this_rq(); @@ -4222,7 +3409,7 @@ static void migrate_dead_tasks(unsigned while (!list_empty(list)) migrate_dead(dead_cpu, list_entry(list->next, task_t, - run_list)); + u.ingosched.run_list)); } } } @@ -4305,7 +3492,7 @@ static struct notifier_block __devinitda .priority = 10 }; -int __init migration_init(void) +static int __init ingo_migration_init(void) { void *cpu = (void *)(long)smp_processor_id(); /* Start one for boot CPU. */ @@ -4321,7 +3508,7 @@ int __init migration_init(void) * Attach the domain 'sd' to 'cpu' as its base domain. Callers must * hold the hotplug lock. */ -void __devinit cpu_attach_domain(struct sched_domain *sd, int cpu) +static void __devinit ingo_cpu_attach_domain(struct sched_domain *sd, int cpu) { migration_req_t req; unsigned long flags; @@ -4348,9 +3535,6 @@ void __devinit cpu_attach_domain(struct } } -/* cpus with isolated domains */ -cpumask_t __devinitdata cpu_isolated_map = CPU_MASK_NONE; - /* Setup the mask of cpus configured for isolated domains */ static int __init isolated_cpu_setup(char *str) { @@ -4365,52 +3549,6 @@ static int __init isolated_cpu_setup(cha __setup ("isolcpus=", isolated_cpu_setup); -/* - * init_sched_build_groups takes an array of groups, the cpumask we wish - * to span, and a pointer to a function which identifies what group a CPU - * belongs to. The return value of group_fn must be a valid index into the - * groups[] array, and must be >= 0 and < NR_CPUS (due to the fact that we - * keep track of groups covered with a cpumask_t). - * - * init_sched_build_groups will build a circular linked list of the groups - * covered by the given span, and will set each group's ->cpumask correctly, - * and ->cpu_power to 0. - */ -void __devinit init_sched_build_groups(struct sched_group groups[], - cpumask_t span, int (*group_fn)(int cpu)) -{ - struct sched_group *first = NULL, *last = NULL; - cpumask_t covered = CPU_MASK_NONE; - int i; - - for_each_cpu_mask(i, span) { - int group = group_fn(i); - struct sched_group *sg = &groups[group]; - int j; - - if (cpu_isset(i, covered)) - continue; - - sg->cpumask = CPU_MASK_NONE; - sg->cpu_power = 0; - - for_each_cpu_mask(j, span) { - if (group_fn(j) != group) - continue; - - cpu_set(j, covered); - cpu_set(j, sg->cpumask); - } - if (!first) - first = sg; - if (last) - last->next = sg; - last = sg; - } - last->next = first; -} - - #ifdef ARCH_HAS_SCHED_DOMAIN extern void __devinit arch_init_sched_domains(void); extern void __devinit arch_destroy_sched_domains(void); @@ -4739,7 +3877,7 @@ static int update_sched_domains(struct n } #endif -void __init sched_init_smp(void) +static void __init ingo_sched_init_smp(void) { lock_cpu_hotplug(); arch_init_sched_domains(); @@ -4749,25 +3887,21 @@ void __init sched_init_smp(void) hotcpu_notifier(update_sched_domains, 0); } #else -void __init sched_init_smp(void) +static void __init ingo_sched_init_smp(void) { } #endif /* CONFIG_SMP */ -int in_sched_functions(unsigned long addr) -{ - /* Linker adds these: start and end of __sched functions */ - extern char __sched_text_start[], __sched_text_end[]; - return in_lock_functions(addr) || - (addr >= (unsigned long)__sched_text_start - && addr < (unsigned long)__sched_text_end); -} - -void __init sched_init(void) +static void __init ingo_sched_init(void) { runqueue_t *rq; int i, j, k; + init_task.u.ingosched.prio = MAX_PRIO - 20; + init_task.static_prio = MAX_PRIO - 20; + INIT_LIST_HEAD(&init_task.u.ingosched.run_list); + init_task.u.ingosched.time_slice = HZ; + for (i = 0; i < NR_CPUS; i++) { prio_array_t *array; @@ -4813,28 +3947,6 @@ void __init sched_init(void) init_idle(current, smp_processor_id()); } -#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP -void __might_sleep(char *file, int line) -{ -#if defined(in_atomic) - static unsigned long prev_jiffy; /* ratelimiting */ - - if ((in_atomic() || irqs_disabled()) && - system_state == SYSTEM_RUNNING) { - if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) - return; - prev_jiffy = jiffies; - printk(KERN_ERR "Debug: sleeping function called from invalid" - " context at %s:%d\n", file, line); - printk("in_atomic():%d, irqs_disabled():%d\n", - in_atomic(), irqs_disabled()); - dump_stack(); - } -#endif -} -EXPORT_SYMBOL(__might_sleep); -#endif - #if defined(CONFIG_DEBUG_KERNEL)&&defined(CONFIG_SYSCTL)&&defined(CONFIG_SMP) static struct ctl_table sd_ctl_dir[] = { {1, "sched_domain", NULL, 0, 0755, NULL, }, @@ -4924,7 +4036,7 @@ static ctl_table *sd_alloc_ctl_cpu_table } static struct ctl_table_header *sd_sysctl_header; -void init_sched_domain_sysctl() +void ingo_init_sched_domain_sysctl(void) { int i, cpu_num = num_online_cpus(); char buf[32]; @@ -4942,7 +4054,7 @@ void init_sched_domain_sysctl() sd_sysctl_header = register_sysctl_table(sd_ctl_root, 0); } -void destroy_sched_domain_sysctl() +static void ingo_destroy_sched_domain_sysctl(void) { int cpu, cpu_num = num_online_cpus(); struct sched_domain *sd; @@ -4964,10 +4076,61 @@ void destroy_sched_domain_sysctl() kfree(root); } #else -void init_sched_domain_sysctl() +static void ingo_init_sched_domain_sysctl(void) { } -void destroy_sched_domain_sysctl() +static void ingo_destroy_sched_domain_sysctl(void) { } #endif + +struct sched_drv ingo_sched_drv = { + .task_cpu = common_task_cpu, + .set_task_cpu = common_set_task_cpu, + .init_sched_domain_sysctl = ingo_init_sched_domain_sysctl, + .destroy_sched_domain_sysctl = ingo_destroy_sched_domain_sysctl, + .account_steal_time = ingo_account_steal_time, + .account_system_time = ingo_account_system_time, + .account_user_time = ingo_account_user_time, + .cpusched_name = "ingosched", + .rt_task = ingo_rt_task, + .wait_for_completion = ingo_wait_for_completion, + .io_schedule = ingo_io_schedule, + .io_schedule_timeout = ingo_io_schedule_timeout, + .set_oom_timeslice = ingo_set_oom_timeslice, + .nr_running = ingo_nr_running, + .nr_uninterruptible = ingo_nr_uninterruptible, + .nr_context_switches = ingo_nr_context_switches, + .nr_iowait = ingo_nr_iowait, + .idle_cpu = ingo_idle_cpu, + .init_idle = ingo_init_idle, + .exit = ingo_sched_exit, + .fork = ingo_sched_fork, + .init = ingo_sched_init, + .init_smp = ingo_sched_init_smp, + .schedule = ingo_schedule, + .tick = ingo_scheduler_tick, + .tail = ingo_schedule_tail, + .setscheduler = ingo_setscheduler, + .set_user_nice = ingo_set_user_nice, + .rr_get_interval = ingo_sys_sched_rr_get_interval, + .yield = ingo_sys_sched_yield, + .task_curr = ingo_task_curr, + .task_nice = ingo_task_nice, + .task_prio = ingo_task_prio, + .try_to_wake_up = ingo_try_to_wake_up, + .wake_up_new_task = ingo_wake_up_new_task, +#ifdef CONFIG_SMP + .migration_init = ingo_migration_init, + .exec = ingo_sched_exec, + .set_cpus_allowed = ingo_set_cpus_allowed, + .wait_task_inactive = ingo_wait_task_inactive, + .cpu_attach_domain = ingo_cpu_attach_domain, +#ifdef CONFIG_HOTPLUG_CPU + .sched_idle_next = ingo_sched_idle_next, +#endif +#endif +#ifdef CONFIG_SCHEDSTATS + .show_schedstat = ingo_show_schedstat, +#endif +}; Index: linux-2.6.10-rc1-mm2-plugsched1/kernel/scheduler.c =================================================================== --- linux-2.6.10-rc1-mm2-plugsched1.orig/kernel/scheduler.c 2003-03-27 19:01:40.000000000 +1100 +++ linux-2.6.10-rc1-mm2-plugsched1/kernel/scheduler.c 2004-11-01 14:50:39.000000000 +1100 @@ -0,0 +1,1246 @@ +/* + * kernel/scheduler.c + * + * Kernel scheduler and related syscalls + * + * Copyright (C) 1991-2002 Linus Torvalds + * + * Modular cpu scheduler infrastructure by Con Kolivas based on + * work by William Lee Irwin III. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +DEFINE_PER_CPU(struct kernel_stat, kstat); +EXPORT_PER_CPU_SYMBOL(kstat); + +unsigned int task_cpu(const struct task_struct *p); + +void set_task_cpu(struct task_struct *p, unsigned int cpu); + +#ifdef CONFIG_SMP +/*** + * kick_process - kick a running thread to enter/exit the kernel + * @p: the to-be-kicked thread + * + * Cause a process which is running on another CPU to enter + * kernel-mode, without any delay. (to get signals handled.) + */ +void kick_process(task_t *p) +{ + int cpu; + + preempt_disable(); + cpu = task_cpu(p); + if ((cpu != smp_processor_id()) && task_curr(p)) + smp_send_reschedule(cpu); + preempt_enable(); +} + +/* + * Wrappers for p->thread_info->cpu access. No-op on UP. + */ +unsigned int common_task_cpu(const struct task_struct *p) +{ + return p->thread_info->cpu; +} + +void common_set_task_cpu(struct task_struct *p, unsigned int cpu) +{ + p->thread_info->cpu = cpu; +} + +#else + +unsigned int common_task_cpu(const struct task_struct *p) +{ + return 0; +} + +void common_set_task_cpu(struct task_struct *p, unsigned int cpu) +{ +} +#endif /* CONFIG_SMP */ + +#ifdef CONFIG_PREEMPT +#ifdef CONFIG_DEBUG_PREEMPT + +void fastcall add_preempt_count(int val) +{ + /* + * Underflow? + */ + BUG_ON(((int)preempt_count() < 0)); + preempt_count() += val; + /* + * Spinlock count overflowing soon? + */ + BUG_ON((preempt_count() & PREEMPT_MASK) >= PREEMPT_MASK-10); +} +EXPORT_SYMBOL(add_preempt_count); + +void fastcall sub_preempt_count(int val) +{ + /* + * Underflow? + */ + BUG_ON(val > preempt_count()); + /* + * Is the spinlock portion underflowing? + */ + BUG_ON((val < PREEMPT_MASK) && !(preempt_count() & PREEMPT_MASK)); + preempt_count() -= val; +} +EXPORT_SYMBOL(sub_preempt_count); + +#endif + +/* + * this is is the entry point to schedule() from in-kernel preemption + * off of preempt_enable. Kernel preemptions off return from interrupt + * occur there and call schedule directly. + */ +asmlinkage void __sched preempt_schedule(void) +{ + struct thread_info *ti = current_thread_info(); +#ifdef CONFIG_PREEMPT_BKL + struct task_struct *task = current; + int saved_lock_depth; +#endif + /* + * If there is a non-zero preempt_count or interrupts are disabled, + * we do not want to preempt the current task. Just return.. + */ + if (unlikely(ti->preempt_count || irqs_disabled())) + return; + +need_resched: + add_preempt_count(PREEMPT_ACTIVE); + /* + * We keep the big kernel semaphore locked, but we + * clear ->lock_depth so that schedule() doesnt + * auto-release the semaphore: + */ +#ifdef CONFIG_PREEMPT_BKL + saved_lock_depth = task->lock_depth; + task->lock_depth = -1; +#endif + schedule(); +#ifdef CONFIG_PREEMPT_BKL + task->lock_depth = saved_lock_depth; +#endif + sub_preempt_count(PREEMPT_ACTIVE); + + /* we could miss a preemption opportunity between schedule and now */ + barrier(); + if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) + goto need_resched; +} + +EXPORT_SYMBOL(preempt_schedule); +#endif /* CONFIG_PREEMPT */ + +#define SLEEP_ON_VAR \ + unsigned long flags; \ + wait_queue_t wait; \ + init_waitqueue_entry(&wait, current); + +#define SLEEP_ON_HEAD \ + spin_lock_irqsave(&q->lock,flags); \ + __add_wait_queue(q, &wait); \ + spin_unlock(&q->lock); + +#define SLEEP_ON_TAIL \ + spin_lock_irq(&q->lock); \ + __remove_wait_queue(q, &wait); \ + spin_unlock_irqrestore(&q->lock, flags); + +void fastcall __sched interruptible_sleep_on(wait_queue_head_t *q) +{ + SLEEP_ON_VAR + + current->state = TASK_INTERRUPTIBLE; + + SLEEP_ON_HEAD + schedule(); + SLEEP_ON_TAIL +} + +EXPORT_SYMBOL(interruptible_sleep_on); + +long fastcall __sched interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout) +{ + SLEEP_ON_VAR + + current->state = TASK_INTERRUPTIBLE; + + SLEEP_ON_HEAD + timeout = schedule_timeout(timeout); + SLEEP_ON_TAIL + + return timeout; +} + +EXPORT_SYMBOL(interruptible_sleep_on_timeout); + +void fastcall __sched sleep_on(wait_queue_head_t *q) +{ + SLEEP_ON_VAR + + current->state = TASK_UNINTERRUPTIBLE; + + SLEEP_ON_HEAD + schedule(); + SLEEP_ON_TAIL +} + +EXPORT_SYMBOL(sleep_on); + +long fastcall __sched sleep_on_timeout(wait_queue_head_t *q, long timeout) +{ + SLEEP_ON_VAR + + current->state = TASK_UNINTERRUPTIBLE; + + SLEEP_ON_HEAD + timeout = schedule_timeout(timeout); + SLEEP_ON_TAIL + + return timeout; +} + +EXPORT_SYMBOL(sleep_on_timeout); + +#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP +void __might_sleep(char *file, int line) +{ +#if defined(in_atomic) + static unsigned long prev_jiffy; /* ratelimiting */ + + if ((in_atomic() || irqs_disabled()) && + system_state == SYSTEM_RUNNING) { + if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) + return; + prev_jiffy = jiffies; + printk(KERN_ERR "Debug: sleeping function called from invalid" + " context at %s:%d\n", file, line); + printk("in_atomic():%d, irqs_disabled():%d\n", + in_atomic(), irqs_disabled()); + dump_stack(); + } +#endif +} +EXPORT_SYMBOL(__might_sleep); +#endif + +#ifdef __ARCH_WANT_SYS_NICE + +/* + * sys_nice - change the priority of the current process. + * @increment: priority increment + * + * sys_setpriority is a more generic, but much slower function that + * does similar things. + */ +asmlinkage long sys_nice(int increment) +{ + int retval; + long nice; + + /* + * Setpriority might change our priority at the same moment. + * We don't have to worry. Conceptually one call occurs first + * and we have a single winner. + */ + if (increment < 0) { + if (!capable(CAP_SYS_NICE)) + return -EPERM; + if (increment < -40) + increment = -40; + } + if (increment > 40) + increment = 40; + + nice = task_nice(current) + increment; + if (nice < -20) + nice = -20; + if (nice > 19) + nice = 19; + + retval = security_task_setnice(current, nice); + if (retval) + return retval; + + set_user_nice(current, nice); + return 0; +} + +#endif + +/** + * find_process_by_pid - find a process with a matching PID value. + * @pid: the pid in question. + */ +task_t *find_process_by_pid(pid_t pid) +{ + return pid ? find_task_by_pid(pid) : current; +} + +int setscheduler(pid_t pid, int policy, struct sched_param __user *param); + +/** + * sys_sched_setscheduler - set/change the scheduler policy and RT priority + * @pid: the pid in question. + * @policy: new policy + * @param: structure containing the new RT priority. + */ +asmlinkage long sys_sched_setscheduler(pid_t pid, int policy, + struct sched_param __user *param) +{ + return setscheduler(pid, policy, param); +} + +/** + * sys_sched_setparam - set/change the RT priority of a thread + * @pid: the pid in question. + * @param: structure containing the new RT priority. + */ +asmlinkage long sys_sched_setparam(pid_t pid, struct sched_param __user *param) +{ + return setscheduler(pid, -1, param); +} + +/** + * sys_sched_getscheduler - get the policy (scheduling class) of a thread + * @pid: the pid in question. + */ +asmlinkage long sys_sched_getscheduler(pid_t pid) +{ + int retval = -EINVAL; + task_t *p; + + if (pid < 0) + goto out_nounlock; + + retval = -ESRCH; + read_lock(&tasklist_lock); + p = find_process_by_pid(pid); + if (p) { + retval = security_task_getscheduler(p); + if (!retval) + retval = p->policy; + } + read_unlock(&tasklist_lock); + +out_nounlock: + return retval; +} + +/** + * sys_sched_getscheduler - get the RT priority of a thread + * @pid: the pid in question. + * @param: structure containing the RT priority. + */ +asmlinkage long sys_sched_getparam(pid_t pid, struct sched_param __user *param) +{ + struct sched_param lp; + int retval = -EINVAL; + task_t *p; + + if (!param || pid < 0) + goto out_nounlock; + + read_lock(&tasklist_lock); + p = find_process_by_pid(pid); + retval = -ESRCH; + if (!p) + goto out_unlock; + + retval = security_task_getscheduler(p); + if (retval) + goto out_unlock; + + lp.sched_priority = p->rt_priority; + read_unlock(&tasklist_lock); + + /* + * This one might sleep, we cannot do it with a spinlock held ... + */ + retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0; + +out_nounlock: + return retval; + +out_unlock: + read_unlock(&tasklist_lock); + return retval; +} + +long sched_setaffinity(pid_t pid, cpumask_t new_mask) +{ + task_t *p; + int retval; + cpumask_t cpus_allowed; + + lock_cpu_hotplug(); + read_lock(&tasklist_lock); + + p = find_process_by_pid(pid); + if (!p) { + read_unlock(&tasklist_lock); + unlock_cpu_hotplug(); + return -ESRCH; + } + + /* + * It is not safe to call set_cpus_allowed with the + * tasklist_lock held. We will bump the task_struct's + * usage count and then drop tasklist_lock. + */ + get_task_struct(p); + read_unlock(&tasklist_lock); + + retval = -EPERM; + if ((current->euid != p->euid) && (current->euid != p->uid) && + !capable(CAP_SYS_NICE)) + goto out_unlock; + + cpus_allowed = cpuset_cpus_allowed(p); + cpus_and(new_mask, new_mask, cpus_allowed); + retval = set_cpus_allowed(p, new_mask); + +out_unlock: + put_task_struct(p); + unlock_cpu_hotplug(); + return retval; +} + +static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, + cpumask_t *new_mask) +{ + if (len < sizeof(cpumask_t)) { + memset(new_mask, 0, sizeof(cpumask_t)); + } else if (len > sizeof(cpumask_t)) { + len = sizeof(cpumask_t); + } + return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0; +} + +/** + * sys_sched_setaffinity - set the cpu affinity of a process + * @pid: pid of the process + * @len: length in bytes of the bitmask pointed to by user_mask_ptr + * @user_mask_ptr: user-space pointer to the new cpu mask + */ +asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len, + unsigned long __user *user_mask_ptr) +{ + cpumask_t new_mask; + int retval; + + retval = get_user_cpu_mask(user_mask_ptr, len, &new_mask); + if (retval) + return retval; + + return sched_setaffinity(pid, new_mask); +} + +/* + * Represents all cpu's present in the system + * In systems capable of hotplug, this map could dynamically grow + * as new cpu's are detected in the system via any platform specific + * method, such as ACPI for e.g. + */ + +cpumask_t cpu_present_map; +EXPORT_SYMBOL(cpu_present_map); + +#ifndef CONFIG_SMP +cpumask_t cpu_online_map = CPU_MASK_ALL; +cpumask_t cpu_possible_map = CPU_MASK_ALL; +#endif + +long sched_getaffinity(pid_t pid, cpumask_t *mask) +{ + int retval; + task_t *p; + + lock_cpu_hotplug(); + read_lock(&tasklist_lock); + + retval = -ESRCH; + p = find_process_by_pid(pid); + if (!p) + goto out_unlock; + + retval = 0; + cpus_and(*mask, p->cpus_allowed, cpu_possible_map); + +out_unlock: + read_unlock(&tasklist_lock); + unlock_cpu_hotplug(); + if (retval) + return retval; + + return 0; +} + +/** + * sys_sched_getaffinity - get the cpu affinity of a process + * @pid: pid of the process + * @len: length in bytes of the bitmask pointed to by user_mask_ptr + * @user_mask_ptr: user-space pointer to hold the current cpu mask + */ +asmlinkage long sys_sched_getaffinity(pid_t pid, unsigned int len, + unsigned long __user *user_mask_ptr) +{ + int ret; + cpumask_t mask; + + if (len < sizeof(cpumask_t)) + return -EINVAL; + + ret = sched_getaffinity(pid, &mask); + if (ret < 0) + return ret; + + if (copy_to_user(user_mask_ptr, &mask, sizeof(cpumask_t))) + return -EFAULT; + + return sizeof(cpumask_t); +} + +/** + * sys_sched_get_priority_max - return maximum RT priority. + * @policy: scheduling class. + * + * this syscall returns the maximum rt_priority that can be used + * by a given scheduling class. + */ +asmlinkage long sys_sched_get_priority_max(int policy) +{ + int ret = -EINVAL; + + switch (policy) { + case SCHED_FIFO: + case SCHED_RR: + ret = MAX_USER_RT_PRIO-1; + break; + case SCHED_NORMAL: + ret = 0; + break; + } + return ret; +} + +/** + * sys_sched_get_priority_min - return minimum RT priority. + * @policy: scheduling class. + * + * this syscall returns the minimum rt_priority that can be used + * by a given scheduling class. + */ +asmlinkage long sys_sched_get_priority_min(int policy) +{ + int ret = -EINVAL; + + switch (policy) { + case SCHED_FIFO: + case SCHED_RR: + ret = 1; + break; + case SCHED_NORMAL: + ret = 0; + } + return ret; +} + +static inline void __cond_resched(void) +{ + do { + add_preempt_count(PREEMPT_ACTIVE); + schedule(); + sub_preempt_count(PREEMPT_ACTIVE); + } while (need_resched()); +} + +int __sched cond_resched(void) +{ + if (need_resched()) { + __cond_resched(); + return 1; + } + return 0; +} + +EXPORT_SYMBOL(cond_resched); + +/* + * cond_resched_lock() - if a reschedule is pending, drop the given lock, + * call schedule, and on return reacquire the lock. + * + * This works OK both with and without CONFIG_PREEMPT. We do strange low-level + * operations here to prevent schedule() from being called twice (once via + * spin_unlock(), once by hand). + */ +int cond_resched_lock(spinlock_t * lock) +{ +#if defined(CONFIG_SMP) && defined(CONFIG_PREEMPT) + if (lock->break_lock) { + lock->break_lock = 0; + spin_unlock(lock); + cpu_relax(); + spin_lock(lock); + } +#endif + if (need_resched()) { + _raw_spin_unlock(lock); + preempt_enable_no_resched(); + __cond_resched(); + spin_lock(lock); + return 1; + } + return 0; +} + +EXPORT_SYMBOL(cond_resched_lock); + +int __sched cond_resched_softirq(void) +{ + BUG_ON(!in_softirq()); + + if (need_resched()) { + __local_bh_enable(); + __cond_resched(); + local_bh_disable(); + return 1; + } + return 0; +} + +EXPORT_SYMBOL(cond_resched_softirq); + +/** + * yield - yield the current processor to other threads. + * + * this is a shortcut for kernel-space yielding - it marks the + * thread runnable and calls sys_sched_yield(). + */ +void __sched yield(void) +{ + set_current_state(TASK_RUNNING); + sys_sched_yield(); +} + +EXPORT_SYMBOL(yield); + +static inline struct task_struct *eldest_child(struct task_struct *p) +{ + if (list_empty(&p->children)) return NULL; + return list_entry(p->children.next,struct task_struct,sibling); +} + +static inline struct task_struct *older_sibling(struct task_struct *p) +{ + if (p->sibling.prev==&p->parent->children) return NULL; + return list_entry(p->sibling.prev,struct task_struct,sibling); +} + +static inline struct task_struct *younger_sibling(struct task_struct *p) +{ + if (p->sibling.next==&p->parent->children) return NULL; + return list_entry(p->sibling.next,struct task_struct,sibling); +} + +static void show_task(task_t * p) +{ + task_t *relative; + unsigned state; + unsigned long free = 0; + static const char *stat_nam[] = { "R", "S", "D", "T", "t", "Z", "X" }; + + printk("%-13.13s ", p->comm); + state = p->state ? __ffs(p->state) + 1 : 0; + if (state < ARRAY_SIZE(stat_nam)) + printk(stat_nam[state]); + else + printk("?"); +#if (BITS_PER_LONG == 32) + if (state == TASK_RUNNING) + printk(" running "); + else + printk(" %08lX ", thread_saved_pc(p)); +#else + if (state == TASK_RUNNING) + printk(" running task "); + else + printk(" %016lx ", thread_saved_pc(p)); +#endif +#ifdef CONFIG_DEBUG_STACK_USAGE + { + unsigned long * n = (unsigned long *) (p->thread_info+1); + while (!*n) + n++; + free = (unsigned long) n - (unsigned long)(p->thread_info+1); + } +#endif + printk("%5lu %5d %6d ", free, p->pid, p->parent->pid); + if ((relative = eldest_child(p))) + printk("%5d ", relative->pid); + else + printk(" "); + if ((relative = younger_sibling(p))) + printk("%7d", relative->pid); + else + printk(" "); + if ((relative = older_sibling(p))) + printk(" %5d", relative->pid); + else + printk(" "); + if (!p->mm) + printk(" (L-TLB)\n"); + else + printk(" (NOTLB)\n"); + + if (state != TASK_RUNNING) + show_stack(p, NULL); +} + +void show_state(void) +{ + task_t *g, *p; + +#if (BITS_PER_LONG == 32) + printk("\n" + " sibling\n"); + printk(" task PC pid father child younger older\n"); +#else + printk("\n" + " sibling\n"); + printk(" task PC pid father child younger older\n"); +#endif + read_lock(&tasklist_lock); + do_each_thread(g, p) { + /* + * reset the NMI-timeout, listing all files on a slow + * console might take alot of time: + */ + touch_nmi_watchdog(); + show_task(p); + } while_each_thread(g, p); + + read_unlock(&tasklist_lock); +} + +/* + * In a system that switches off the HZ timer nohz_cpu_mask + * indicates which cpus entered this state. This is used + * in the rcu update to wait only for active cpus. For system + * which do not switch off the HZ timer nohz_cpu_mask should + * always be CPU_MASK_NONE. + */ +cpumask_t nohz_cpu_mask = CPU_MASK_NONE; + +int in_sched_functions(unsigned long addr) +{ + /* Linker adds these: start and end of __sched functions */ + extern char __sched_text_start[], __sched_text_end[]; + return in_lock_functions(addr) || + (addr >= (unsigned long)__sched_text_start + && addr < (unsigned long)__sched_text_end); +} + +int try_to_wake_up(task_t *task, unsigned state, int sync); + +int fastcall wake_up_state(task_t *p, unsigned int state) +{ + return try_to_wake_up(p, state, 0); +} + +int fastcall wake_up_process(task_t * p) +{ + return try_to_wake_up(p, TASK_STOPPED | TASK_TRACED | + TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE, 0); +} +EXPORT_SYMBOL(wake_up_process); + +/* + * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just + * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve + * number) then we wake all the non-exclusive tasks and one exclusive task. + * + * There are circumstances in which we can try to wake a task which has already + * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns + * zero in this (rare) case, and we handle it by continuing to scan the queue. + */ +static void __wake_up_common(wait_queue_head_t *q, unsigned int mode, + int nr_exclusive, int sync, void *key) +{ + struct list_head *tmp, *next; + + list_for_each_safe(tmp, next, &q->task_list) { + wait_queue_t *curr; + unsigned flags; + curr = list_entry(tmp, wait_queue_t, task_list); + flags = curr->flags; + if (curr->func(curr, mode, sync, key) && + (flags & WQ_FLAG_EXCLUSIVE) && + !--nr_exclusive) + break; + } +} + +/** + * __wake_up - wake up threads blocked on a waitqueue. + * @q: the waitqueue + * @mode: which threads + * @nr_exclusive: how many wake-one or wake-many threads to wake up + */ +void fastcall __wake_up(wait_queue_head_t *q, unsigned int mode, + int nr_exclusive, void *key) +{ + unsigned long flags; + + spin_lock_irqsave(&q->lock, flags); + __wake_up_common(q, mode, nr_exclusive, 0, key); + spin_unlock_irqrestore(&q->lock, flags); +} +EXPORT_SYMBOL(__wake_up); + +int default_wake_function(wait_queue_t *curr, unsigned mode, int sync, void *key) +{ + task_t *p = curr->task; + return try_to_wake_up(p, mode, sync); +} +EXPORT_SYMBOL(default_wake_function); + +/* + * Same as __wake_up but called with the spinlock in wait_queue_head_t held. + */ +void fastcall __wake_up_locked(wait_queue_head_t *q, unsigned int mode) +{ + __wake_up_common(q, mode, 1, 0, NULL); +} + +/** + * __wake_up - sync- wake up threads blocked on a waitqueue. + * @q: the waitqueue + * @mode: which threads + * @nr_exclusive: how many wake-one or wake-many threads to wake up + * + * The sync wakeup differs that the waker knows that it will schedule + * away soon, so while the target thread will be woken up, it will not + * be migrated to another CPU - ie. the two threads are 'synchronized' + * with each other. This can prevent needless bouncing between CPUs. + * + * On UP it can prevent extra preemption. + */ +void fastcall __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive) +{ + unsigned long flags; + int sync = 1; + + if (unlikely(!q)) + return; + + if (unlikely(!nr_exclusive)) + sync = 0; + + spin_lock_irqsave(&q->lock, flags); + __wake_up_common(q, mode, nr_exclusive, sync, NULL); + spin_unlock_irqrestore(&q->lock, flags); +} +EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */ + +void fastcall complete(struct completion *x) +{ + unsigned long flags; + + spin_lock_irqsave(&x->wait.lock, flags); + x->done++; + __wake_up_common(&x->wait, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, + 1, 0, NULL); + spin_unlock_irqrestore(&x->wait.lock, flags); +} +EXPORT_SYMBOL(complete); + +void fastcall complete_all(struct completion *x) +{ + unsigned long flags; + + spin_lock_irqsave(&x->wait.lock, flags); + x->done += UINT_MAX/2; + __wake_up_common(&x->wait, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, + 0, 0, NULL); + spin_unlock_irqrestore(&x->wait.lock, flags); +} +EXPORT_SYMBOL(complete_all); + +#ifdef CONFIG_SMP +/* cpus with isolated domains */ +cpumask_t __devinitdata cpu_isolated_map = CPU_MASK_NONE; + +/* + * init_sched_build_groups takes an array of groups, the cpumask we wish + * to span, and a pointer to a function which identifies what group a CPU + * belongs to. The return value of group_fn must be a valid index into the + * groups[] array, and must be >= 0 and < NR_CPUS (due to the fact that we + * keep track of groups covered with a cpumask_t). + * + * init_sched_build_groups will build a circular linked list of the groups + * covered by the given span, and will set each group's ->cpumask correctly, + * and ->cpu_power to 0. + */ +void __devinit init_sched_build_groups(struct sched_group groups[], + cpumask_t span, int (*group_fn)(int cpu)) +{ + struct sched_group *first = NULL, *last = NULL; + cpumask_t covered = CPU_MASK_NONE; + int i; + + for_each_cpu_mask(i, span) { + int group = group_fn(i); + struct sched_group *sg = &groups[group]; + int j; + + if (cpu_isset(i, covered)) + continue; + + sg->cpumask = CPU_MASK_NONE; + sg->cpu_power = 0; + + for_each_cpu_mask(j, span) { + if (group_fn(j) != group) + continue; + + cpu_set(j, covered); + cpu_set(j, sg->cpumask); + } + if (!first) + first = sg; + if (last) + last->next = sg; + last = sg; + } + last->next = first; +} +#endif + +#ifdef CONFIG_SCHEDSTATS +int show_schedstat(struct seq_file *seq, void *v); + +static int schedstat_open(struct inode *inode, struct file *file) +{ + unsigned int size = PAGE_SIZE * (1 + num_online_cpus() / 32); + char *buf = kmalloc(size, GFP_KERNEL); + struct seq_file *m; + int res; + + if (!buf) + return -ENOMEM; + res = single_open(file, show_schedstat, NULL); + if (!res) { + m = file->private_data; + m->buf = buf; + m->size = size; + } else + kfree(buf); + return res; +} + +struct file_operations proc_schedstat_operations = { + .open = schedstat_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; +#endif + +extern struct sched_drv ingo_sched_drv; +extern struct sched_drv sc_sched_drv; + +struct sched_drv *scheduler = +#if defined(CONFIG_CPUSCHED_DEFAULT_INGO) + &ingo_sched_drv; +#elif defined(CONFIG_CPUSCHED_DEFAULT_STAIRCASE) + &sc_sched_drv; +#else + NULL; +#error "You must have at least 1 cpu scheduler selected" +#endif + +static int __init scheduler_setup(char *str) +{ + struct sched_drv *chosen_sched = NULL; +#if defined(CONFIG_CPUSCHED_INGO) + if (!strcmp(str, ingo_sched_drv.cpusched_name)) + chosen_sched = &ingo_sched_drv; +#endif +#if defined(CONFIG_CPUSCHED_STAIRCASE) + if (!strcmp(str, sc_sched_drv.cpusched_name)) + chosen_sched = &sc_sched_drv; +#endif + if (chosen_sched && chosen_sched != scheduler) { + /* + * A different cpu scheduler from the default has been + * chosen. We need to reinit the scheduler. Set the scheduler + * pointer to the new chosen scheduler. + */ + scheduler = chosen_sched; + /* Get a fresh init_task from the saved one */ + init_task = base_init_task; + /* Repeat sched_init sequence */ + sched_init(); + preempt_disable(); + } + return 1; +} + +__setup ("cpusched=", scheduler_setup); + +unsigned int task_cpu(const struct task_struct *p) +{ + return scheduler->task_cpu(p); +} + +void set_task_cpu(struct task_struct *p, unsigned int cpu) +{ + scheduler->set_task_cpu(p, cpu); +} + +void init_sched_domain_sysctl(void) +{ + scheduler->init_sched_domain_sysctl(); +} + +void destroy_sched_domain_sysctl(void) +{ + scheduler->destroy_sched_domain_sysctl(); +} + +void account_steal_time(struct task_struct *p, cputime_t steal) +{ + scheduler->account_steal_time(p, steal); +} + +void account_system_time(struct task_struct *p, int hardirq_offset, + cputime_t cputime) +{ + scheduler->account_system_time(p, hardirq_offset, cputime); +} + +void account_user_time(struct task_struct *p, cputime_t cputime) +{ + scheduler->account_user_time(p, cputime); +} + +void fastcall __sched wait_for_completion(struct completion *x) +{ + scheduler->wait_for_completion(x); +} + +void sched_idle_next(void) +{ + scheduler->sched_idle_next(); +} + +void __sched io_schedule(void) +{ + scheduler->io_schedule(); +} +EXPORT_SYMBOL(io_schedule); + +long __sched io_schedule_timeout(long timeout) +{ + return scheduler->io_schedule_timeout(timeout); +} + +unsigned long nr_running(void) +{ + return scheduler->nr_running(); +} + +unsigned long nr_uninterruptible(void) +{ + return scheduler->nr_uninterruptible(); +} + +unsigned long long nr_context_switches(void) +{ + return scheduler->nr_context_switches(); +} + +unsigned long nr_iowait(void) +{ + return scheduler->nr_iowait(); +} + +int rt_task(task_t *task) +{ + return scheduler->rt_task(task); +} + +int idle_cpu(int cpu) +{ + return scheduler->idle_cpu(cpu); +} +EXPORT_SYMBOL_GPL(idle_cpu); + +void __devinit init_idle(task_t *task, int cpu) +{ + scheduler->init_idle(task, cpu); +} + +void __init sched_init(void) +{ + scheduler->init(); +} + +void __init sched_init_smp(void) +{ + scheduler->init_smp(); +} + +asmlinkage void schedule(void) +{ + scheduler->schedule(); +} +EXPORT_SYMBOL(schedule); + +void scheduler_tick(void) +{ + scheduler->tick(); +} + +#ifdef CONFIG_SMP +int migration_init(void) +{ + return scheduler->migration_init(); +} + +int set_cpus_allowed(task_t *task, cpumask_t cpus) +{ + return scheduler->set_cpus_allowed(task, cpus); +} +EXPORT_SYMBOL_GPL(set_cpus_allowed); + +void wait_task_inactive(task_t * task) +{ + scheduler->wait_task_inactive(task); +} + +void sched_exec(void) +{ + scheduler->exec(); +} + +void __devinit cpu_attach_domain(struct sched_domain *sd, int cpu) +{ + scheduler->cpu_attach_domain(sd, cpu); +} +#endif + +void set_user_nice(task_t *task, long nice) +{ + scheduler->set_user_nice(task, nice); +} +EXPORT_SYMBOL(set_user_nice); + +void set_oom_timeslice(task_t *p) +{ + scheduler->set_oom_timeslice(p); +} + +asmlinkage +long sys_sched_rr_get_interval(pid_t pid, struct timespec __user *interval) +{ + return scheduler->rr_get_interval(pid, interval); +} + +asmlinkage long sys_sched_yield(void) +{ + return scheduler->yield(); +} + +int setscheduler(pid_t pid, int policy, struct sched_param __user *param) +{ + return scheduler->setscheduler(pid, policy, param); +} + +int task_curr(const task_t *task) +{ + return scheduler->task_curr(task); +} + +int task_nice(const task_t *task) +{ + return scheduler->task_nice(task); +} +EXPORT_SYMBOL(task_nice); + +int task_prio(const task_t *task) +{ + return scheduler->task_prio(task); +} + +int try_to_wake_up(task_t *task, unsigned state, int sync) +{ + return scheduler->try_to_wake_up(task, state, sync); +} + +void fastcall wake_up_new_task(task_t *task, unsigned long flags) +{ + scheduler->wake_up_new_task(task, flags); +} + +void fastcall sched_fork(task_t *task) +{ + scheduler->fork(task); +} + +void fastcall sched_exit(task_t *task) +{ + scheduler->exit(task); +} + +asmlinkage void schedule_tail(task_t *task) +{ + scheduler->tail(task); +} + +#ifdef CONFIG_SCHEDSTATS +int show_schedstat(struct seq_file *seq, void *v) +{ + return scheduler->show_schedstat(seq, v); +} +#endif Index: linux-2.6.10-rc1-mm2-plugsched1/kernel/staircase.c =================================================================== --- linux-2.6.10-rc1-mm2-plugsched1.orig/kernel/staircase.c 2003-03-27 19:01:40.000000000 +1100 +++ linux-2.6.10-rc1-mm2-plugsched1/kernel/staircase.c 2004-11-01 14:50:39.000000000 +1100 @@ -0,0 +1,3875 @@ +/* + * kernel/staircase.c + * + * This is the staircase cpu scheduler + * + * Copyright (C) 1991-2002 Linus Torvalds + * + * 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and + * make semaphores SMP safe + * 1998-11-19 Implemented schedule_timeout() and related stuff + * by Andrea Arcangeli + * 2002-01-04 New ultra-scalable O(1) scheduler by Ingo Molnar: + * hybrid priority-list and round-robin design with + * an array-switch method of distributing timeslices + * and per-CPU runqueues. Cleanups and useful suggestions + * by Davide Libenzi, preemptible kernel bits by Robert Love. + * 2004-04-02 Scheduler domains code by Nick Piggin + * 2004-10-28 New staircase scheduling policy by Con Kolivas with help + * from William Lee Irwin III, Zwane Mwaikambo & Peter Williams. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +/* + * Unique staircase process flags used by scheduler. + */ +#define SF_FORKED 0x00000001 /* I have just forked */ +#define SF_YIELDED 0x00000002 /* I have just yielded */ +#define SF_UISLEEP 0x00000004 /* Uninterruptible sleep */ + +/* + * Priority of a process goes from 0..MAX_PRIO-1, valid RT + * priority is 0..MAX_RT_PRIO-1, and SCHED_NORMAL tasks are + * in the range MAX_RT_PRIO..MAX_PRIO-1. Priority values + * are inverted: lower p->u.scsched.prio value means higher priority. + */ + +#define MAX_PRIO (MAX_RT_PRIO + 40) + +/* + * Convert user-nice values [ -20 ... 0 ... 19 ] + * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ], + * and back. + */ +#define NICE_TO_PRIO(nice) (MAX_RT_PRIO + (nice) + 20) +#define PRIO_TO_NICE(prio) ((prio) - MAX_RT_PRIO - 20) +#define TASK_NICE(p) PRIO_TO_NICE((p)->static_prio) + +/* + * 'User priority' is the nice value converted to something we + * can work with better when scaling various scheduler parameters, + * it's a [ 0 ... 39 ] range. + */ +#define USER_PRIO(p) ((p)-MAX_RT_PRIO) +#define TASK_USER_PRIO(p) USER_PRIO((p)->static_prio) +#define MAX_USER_PRIO (USER_PRIO(MAX_PRIO)) + +/* + * Some helpers for converting nanosecond timing to jiffy resolution + */ +#define NS_TO_JIFFIES(TIME) ((TIME) / (1000000000 / HZ)) +#define NSJIFFY (1000000000 / HZ) /* One jiffy in ns */ + +int sched_compute = 0; +/* + *This is the time all tasks within the same priority round robin. + *compute setting is reserved for dedicated computational scheduling + *and has ten times larger intervals. + */ +#define _RR_INTERVAL ((10 * HZ / 1000) ? : 1) +#define RR_INTERVAL() (_RR_INTERVAL * (1 + 9 * sched_compute)) + +#define task_hot(p, now, sd) ((long long) \ + ((now) - (p)->u.scsched.timestamp) < \ + (long long) (sd)->cache_hot_time) + +/* + * These are the runqueue data structures: + */ + +typedef struct runqueue runqueue_t; + +/* + * This is the main, per-CPU runqueue data structure. + * + * Locking rule: those places that want to lock multiple runqueues + * (such as the load balancing or the thread migration code), lock + * acquire operations must be ordered by ascending &runqueue. + */ +struct runqueue { + spinlock_t lock; + + /* + * nr_running and cpu_load should be in the same cacheline because + * remote CPUs use both these fields when doing load calculation. + */ + unsigned long nr_running; +#ifdef CONFIG_SMP + unsigned long cpu_load; +#endif + unsigned long long nr_switches; + unsigned long nr_uninterruptible; + unsigned long long timestamp_last_tick; + unsigned int cache_ticks, preempted; + task_t *curr, *idle; + struct mm_struct *prev_mm; + unsigned long bitmap[BITS_TO_LONGS(MAX_PRIO+1)]; + struct list_head queue[MAX_PRIO + 1]; + atomic_t nr_iowait; + +#ifdef CONFIG_SMP + struct sched_domain *sd; + + /* For active balancing */ + int active_balance; + int push_cpu; + + task_t *migration_thread; + struct list_head migration_queue; +#endif + +#ifdef CONFIG_SCHEDSTATS + /* latency stats */ + struct sched_info rq_sched_info; + + /* sys_sched_yield() stats */ + unsigned long yld_exp_empty; + unsigned long yld_act_empty; + unsigned long yld_both_empty; + unsigned long yld_cnt; + + /* schedule() stats */ + unsigned long sched_noswitch; + unsigned long sched_switch; + unsigned long sched_cnt; + unsigned long sched_goidle; + + /* pull_task() stats */ + unsigned long pt_gained[MAX_IDLE_TYPES]; + unsigned long pt_lost[MAX_IDLE_TYPES]; + + /* active_load_balance() stats */ + unsigned long alb_cnt; + unsigned long alb_lost; + unsigned long alb_gained; + unsigned long alb_failed; + + /* try_to_wake_up() stats */ + unsigned long ttwu_cnt; + unsigned long ttwu_attempts; + unsigned long ttwu_moved; + + /* wake_up_new_task() stats */ + unsigned long wunt_cnt; + unsigned long wunt_moved; + + /* sched_migrate_task() stats */ + unsigned long smt_cnt; + + /* sched_balance_exec() stats */ + unsigned long sbe_cnt; +#endif +}; + +static DEFINE_PER_CPU(struct runqueue, runqueues); + +#define for_each_domain(cpu, domain) \ + for (domain = cpu_rq(cpu)->sd; domain; domain = domain->parent) + +#define cpu_rq(cpu) (&per_cpu(runqueues, (cpu))) +#define this_rq() (&__get_cpu_var(runqueues)) +#define task_rq(p) cpu_rq(task_cpu(p)) +#define cpu_curr(cpu) (cpu_rq(cpu)->curr) + +static int sc_rt_task(task_t *p) +{ + return (unlikely((p)->u.scsched.prio < MAX_RT_PRIO)); +} + +/* + * Default context-switch locking: + */ +#ifndef prepare_arch_switch +# define prepare_arch_switch(rq, next) do { } while (0) +# define finish_arch_switch(rq, next) spin_unlock_irq(&(rq)->lock) +# define task_running(rq, p) ((rq)->curr == (p)) +#endif + +/* + * task_rq_lock - lock the runqueue a given task resides on and disable + * interrupts. Note the ordering: we can safely lookup the task_rq without + * explicitly disabling preemption. + */ +static runqueue_t *task_rq_lock(task_t *p, unsigned long *flags) +{ + struct runqueue *rq; + +repeat_lock_task: + local_irq_save(*flags); + rq = task_rq(p); + spin_lock(&rq->lock); + if (unlikely(rq != task_rq(p))) { + spin_unlock_irqrestore(&rq->lock, *flags); + goto repeat_lock_task; + } + return rq; +} + +static inline void task_rq_unlock(runqueue_t *rq, unsigned long *flags) +{ + spin_unlock_irqrestore(&rq->lock, *flags); +} + +#ifdef CONFIG_SCHEDSTATS +/* + * bump this up when changing the output format or the meaning of an existing + * format, so that tools can adapt (or abort) + */ +#define SCHEDSTAT_VERSION 10 + +static int sc_show_schedstat(struct seq_file *seq, void *v) +{ + int cpu; + enum idle_type itype; + + seq_printf(seq, "version %d\n", SCHEDSTAT_VERSION); + seq_printf(seq, "timestamp %lu\n", jiffies); + for_each_online_cpu(cpu) { + runqueue_t *rq = cpu_rq(cpu); +#ifdef CONFIG_SMP + struct sched_domain *sd; + int dcnt = 0; +#endif + + /* runqueue-specific stats */ + seq_printf(seq, + "cpu%d %lu %lu %lu %lu %lu %lu %lu %lu %lu %lu %lu %lu " + "%lu %lu %lu %lu %lu %lu %lu %lu %lu %lu", + cpu, rq->yld_both_empty, + rq->yld_act_empty, rq->yld_exp_empty, + rq->yld_cnt, rq->sched_noswitch, + rq->sched_switch, rq->sched_cnt, rq->sched_goidle, + rq->alb_cnt, rq->alb_gained, rq->alb_lost, + rq->alb_failed, + rq->ttwu_cnt, rq->ttwu_moved, rq->ttwu_attempts, + rq->wunt_cnt, rq->wunt_moved, + rq->smt_cnt, rq->sbe_cnt, rq->rq_sched_info.cpu_time, + rq->rq_sched_info.run_delay, rq->rq_sched_info.pcnt); + + for (itype = SCHED_IDLE; itype < MAX_IDLE_TYPES; itype++) + seq_printf(seq, " %lu %lu", rq->pt_gained[itype], + rq->pt_lost[itype]); + seq_printf(seq, "\n"); + +#ifdef CONFIG_SMP + /* domain-specific stats */ + for_each_domain(cpu, sd) { + char mask_str[NR_CPUS]; + + cpumask_scnprintf(mask_str, NR_CPUS, sd->span); + seq_printf(seq, "domain%d %s", dcnt++, mask_str); + for (itype = SCHED_IDLE; itype < MAX_IDLE_TYPES; + itype++) { + seq_printf(seq, " %lu %lu %lu %lu %lu", + sd->lb_cnt[itype], + sd->lb_failed[itype], + sd->lb_imbalance[itype], + sd->lb_nobusyq[itype], + sd->lb_nobusyg[itype]); + } + seq_printf(seq, " %lu %lu %lu %lu\n", + sd->sbe_pushed, sd->sbe_attempts, + sd->ttwu_wake_affine, sd->ttwu_wake_balance); + } +#endif + } + return 0; +} + +# define schedstat_inc(rq, field) rq->field++; +# define schedstat_add(rq, field, amt) rq->field += amt; +#else /* !CONFIG_SCHEDSTATS */ +# define schedstat_inc(rq, field) do { } while (0); +# define schedstat_add(rq, field, amt) do { } while (0); +#endif + +/* + * rq_lock - lock a given runqueue and disable interrupts. + */ +static runqueue_t *this_rq_lock(void) +{ + runqueue_t *rq; + + local_irq_disable(); + rq = this_rq(); + spin_lock(&rq->lock); + + return rq; +} + +static inline void rq_unlock(runqueue_t *rq) +{ + spin_unlock_irq(&rq->lock); +} + +#ifdef CONFIG_SCHED_SMT +static int cpu_and_siblings_are_idle(int cpu) +{ + int sib; + for_each_cpu_mask(sib, cpu_sibling_map[cpu]) { + if (idle_cpu(sib)) + continue; + return 0; + } + + return 1; +} +#else +#define cpu_and_siblings_are_idle(A) idle_cpu(A) +#endif + +#ifdef CONFIG_SCHEDSTATS +/* + * Called when a process is dequeued from the active array and given + * the cpu. We should note that with the exception of interactive + * tasks, the expired queue will become the active queue after the active + * queue is empty, without explicitly dequeuing and requeuing tasks in the + * expired queue. (Interactive tasks may be requeued directly to the + * active queue, thus delaying tasks in the expired queue from running; + * see scheduler_tick()). + * + * This function is only called from sched_info_arrive(), rather than + * dequeue_task(). Even though a task may be queued and dequeued multiple + * times as it is shuffled about, we're really interested in knowing how + * long it was from the *first* time it was queued to the time that it + * finally hit a cpu. + */ +static inline void sched_info_dequeued(task_t *t) +{ + t->sched_info.last_queued = 0; +} + +/* + * Called when a task finally hits the cpu. We can now calculate how + * long it was waiting to run. We also note when it began so that we + * can keep stats on how long its timeslice is. + */ +static inline void sched_info_arrive(task_t *t) +{ + unsigned long now = jiffies, diff = 0; + struct runqueue *rq = task_rq(t); + + if (t->sched_info.last_queued) + diff = now - t->sched_info.last_queued; + sched_info_dequeued(t); + t->sched_info.run_delay += diff; + t->sched_info.last_arrival = now; + t->sched_info.pcnt++; + + if (!rq) + return; + + rq->rq_sched_info.run_delay += diff; + rq->rq_sched_info.pcnt++; +} + +/* + * Called when a process is queued into either the active or expired + * array. The time is noted and later used to determine how long we + * had to wait for us to reach the cpu. Since the expired queue will + * become the active queue after active queue is empty, without dequeuing + * and requeuing any tasks, we are interested in queuing to either. It + * is unusual but not impossible for tasks to be dequeued and immediately + * requeued in the same or another array: this can happen in sched_yield(), + * set_user_nice(), and even load_balance() as it moves tasks from runqueue + * to runqueue. + * + * This function is only called from enqueue_task(), but also only updates + * the timestamp if it is already not set. It's assumed that + * sched_info_dequeued() will clear that stamp when appropriate. + */ +static inline void sched_info_queued(task_t *t) +{ + if (!t->sched_info.last_queued) + t->sched_info.last_queued = jiffies; +} + +/* + * Called when a process ceases being the active-running process, either + * voluntarily or involuntarily. Now we can calculate how long we ran. + */ +static inline void sched_info_depart(task_t *t) +{ + struct runqueue *rq = task_rq(t); + unsigned long diff = jiffies - t->sched_info.last_arrival; + + t->sched_info.cpu_time += diff; + + if (rq) + rq->rq_sched_info.cpu_time += diff; +} + +/* + * Called when tasks are switched involuntarily due, typically, to expiring + * their time slice. (This may also be called when switching to or from + * the idle task.) We are only called when prev != next. + */ +static inline void sched_info_switch(task_t *prev, task_t *next) +{ + struct runqueue *rq = task_rq(prev); + + /* + * prev now departs the cpu. It's not interesting to record + * stats about how efficient we were at scheduling the idle + * process, however. + */ + if (prev != rq->idle) + sched_info_depart(prev); + + if (next != rq->idle) + sched_info_arrive(next); +} +#else +#define sched_info_queued(t) do { } while (0) +#define sched_info_switch(t, next) do { } while (0) +#endif /* CONFIG_SCHEDSTATS */ + +/* + * Get nanosecond clock difference without overflowing unsigned long. + */ +static unsigned long ns_diff(unsigned long long v1, unsigned long long v2) +{ + unsigned long long vdiff; + if (unlikely(v1 < v2)) + /* + * Rarely the clock goes backwards. There should always be + * a positive difference so return 1. + */ + vdiff = 1; + else + vdiff = v1 - v2; + if (vdiff > (1 << 31)) + vdiff = 1 << 31; + return (unsigned long)vdiff; +} + +static inline int task_queued(task_t *task) +{ + return !list_empty(&task->u.scsched.run_list); +} + +/* + * Adding/removing a task to/from a runqueue: + */ +static void dequeue_task(struct task_struct *p, runqueue_t *rq) +{ + list_del_init(&p->u.scsched.run_list); + if (list_empty(rq->queue + p->u.scsched.prio)) + __clear_bit(p->u.scsched.prio, rq->bitmap); + p->u.scsched.ns_debit = 0; +} + +static void enqueue_task(struct task_struct *p, runqueue_t *rq) +{ + list_add_tail(&p->u.scsched.run_list, rq->queue + p->u.scsched.prio); + __set_bit(p->u.scsched.prio, rq->bitmap); +} + +/* + * Used by the migration code - we pull tasks from the head of the + * remote queue so we want these tasks to show up at the head of the + * local queue: + */ +static void enqueue_task_head(struct task_struct *p, runqueue_t *rq) +{ + list_add(&p->u.scsched.run_list, rq->queue + p->u.scsched.prio); + __set_bit(p->u.scsched.prio, rq->bitmap); +} + +/* + * __activate_task - move a task to the runqueue. + */ +static void __activate_task(task_t *p, runqueue_t *rq) +{ + enqueue_task(p, rq); + rq->nr_running++; +} + +/* + * __activate_idle_task - move idle task to the _front_ of runqueue. + */ +static inline void __activate_idle_task(task_t *p, runqueue_t *rq) +{ + enqueue_task_head(p, rq); + rq->nr_running++; +} + +/* + * burst - extra intervals an interactive task can run for at best priority + * instead of descending priorities. + */ +static unsigned int burst(task_t *p) +{ + if (likely(!rt_task(p))) { + unsigned int task_user_prio = TASK_USER_PRIO(p); + return 39 - task_user_prio; + } else + return p->u.scsched.burst; +} + +static void inc_burst(task_t *p) +{ + unsigned int best_burst; + best_burst = burst(p); + if (p->u.scsched.burst < best_burst) + p->u.scsched.burst++; +} + +static void dec_burst(task_t *p) +{ + if (p->u.scsched.burst) + p->u.scsched.burst--; +} + +/* + * slice - the duration a task runs before getting requeued at it's best + * priority and has it's burst decremented. + */ +static unsigned int slice(task_t *p) +{ + unsigned int slice = RR_INTERVAL(); + if (likely(!rt_task(p))) + slice += burst(p) * RR_INTERVAL(); + return slice; +} + +/* + * sched_interactive - sysctl which allows interactive tasks to have bursts + */ +int sched_interactive = 1; + +static unsigned int rr_interval(task_t * p) +{ + unsigned int rr_interval = RR_INTERVAL(); + int nice = TASK_NICE(p); + + if (nice < 0 && !rt_task(p)) + rr_interval += -(nice); + return rr_interval; +} + +static void sc_set_oom_timeslice(task_t *p) +{ + p->u.scsched.slice = slice(p) * 10; + p->u.scsched.time_slice = p->u.scsched.slice; +} + +/* + * effective_prio - dynamic priority dependent on burst. + * The priority normally decreases by one each RR_INTERVAL. + * As the burst increases the priority stays at the top "stair" or + * priority for longer. + */ +static int effective_prio(task_t *p) +{ + int prio; + unsigned int full_slice, used_slice, first_slice; + unsigned int best_burst, rr; + if (rt_task(p)) + return p->u.scsched.prio; + + best_burst = burst(p); + full_slice = slice(p); + rr = rr_interval(p); + used_slice = full_slice - p->u.scsched.slice; + if (p->u.scsched.burst > best_burst) + p->u.scsched.burst = best_burst; + first_slice = rr; + if (sched_interactive && !sched_compute && p->mm) + first_slice *= (p->u.scsched.burst + 1); + prio = MAX_PRIO - 1 - best_burst; + + if (used_slice < first_slice) + return prio; + prio += 1 + (used_slice - first_slice) / rr; + if (prio > MAX_PRIO - 1) + prio = MAX_PRIO - 1; + return prio; +} + +/* + * recalc_task_prio - this checks for tasks that run ultra short timeslices + * or have just forked a thread/process and make them continue their old + * slice instead of starting a new one at high priority. + */ +static void recalc_task_prio(task_t *p, unsigned long long now) +{ + unsigned long sleep_time = ns_diff(now, p->u.scsched.timestamp); + unsigned int rr = rr_interval(p); + unsigned int best_burst = burst(p); + unsigned int minrun = rr * (p->u.scsched.burst + 1) / (best_burst + 1) ? : 1; + if (p->u.scsched.sflags & SF_FORKED || + (NS_TO_JIFFIES(p->u.scsched.runtime + sleep_time) < minrun || + ((!sched_interactive || sched_compute) && + NS_TO_JIFFIES(p->u.scsched.runtime + sleep_time) < rr))) { + unsigned long ns_totalrun = p->u.scsched.totalrun + p->u.scsched.runtime; + unsigned long total_run = NS_TO_JIFFIES(ns_totalrun); + p->u.scsched.sflags &= ~SF_FORKED; + if (p->u.scsched.slice - total_run < 1) { + p->u.scsched.totalrun = 0; + dec_burst(p); + } else { + unsigned int intervals = total_run / rr; + unsigned int remainder; + p->u.scsched.totalrun = ns_totalrun; + p->u.scsched.slice -= intervals * rr; + if (p->u.scsched.slice <= rr) { + p->u.scsched.totalrun = 0; + dec_burst(p); + } else { + remainder = p->u.scsched.slice % rr; + if (remainder) + p->u.scsched.time_slice = remainder; + } + } + } else { + if (NS_TO_JIFFIES(p->u.scsched.totalrun) > (best_burst - p->u.scsched.burst) * rr) + dec_burst(p); + else if (!(p->u.scsched.sflags & SF_UISLEEP || p->u.scsched.totalrun)) + inc_burst(p); + p->u.scsched.runtime = 0; + p->u.scsched.totalrun = 0; + } +} + +/* + * activate_task - move a task to the runqueue and do priority recalculation + * + * Update all the scheduling statistics stuff. (sleep average + * calculation, priority modifiers, etc.) + */ +static void activate_task(task_t *p, runqueue_t *rq, int local) +{ + unsigned long long now = sched_clock(); +#ifdef CONFIG_SMP + if (!local) { + /* Compensate for drifting sched_clock */ + runqueue_t *this_rq = this_rq(); + now = (now - this_rq->timestamp_last_tick) + + rq->timestamp_last_tick; + } +#endif + p->u.scsched.slice = slice(p); + p->u.scsched.time_slice = rr_interval(p); + recalc_task_prio(p, now); + p->u.scsched.sflags &= ~SF_UISLEEP; + p->u.scsched.prio = effective_prio(p); + p->u.scsched.timestamp = now; + __activate_task(p, rq); +} + +/* + * deactivate_task - remove a task from the runqueue. + */ +static void deactivate_task(struct task_struct *p, runqueue_t *rq) +{ + rq->nr_running--; + if (p->state == TASK_UNINTERRUPTIBLE) { + p->u.scsched.sflags |= SF_UISLEEP; + rq->nr_uninterruptible++; + } + dequeue_task(p, rq); +} + +/* + * resched_task - mark a task 'to be rescheduled now'. + * + * On UP this means the setting of the need_resched flag, on SMP it + * might also involve a cross-CPU call to trigger the scheduler on + * the target CPU. + */ +#ifdef CONFIG_SMP +static void resched_task(task_t *p) +{ + int need_resched, nrpolling; + + BUG_ON(!spin_is_locked(&task_rq(p)->lock)); + + /* minimise the chance of sending an interrupt to poll_idle() */ + nrpolling = test_tsk_thread_flag(p,TIF_POLLING_NRFLAG); + need_resched = test_and_set_tsk_thread_flag(p,TIF_NEED_RESCHED); + nrpolling |= test_tsk_thread_flag(p,TIF_POLLING_NRFLAG); + + if (!need_resched && !nrpolling && (task_cpu(p) != smp_processor_id())) + smp_send_reschedule(task_cpu(p)); +} +#else +static inline void resched_task(task_t *p) +{ + set_tsk_need_resched(p); +} +#endif + +/** + * task_curr - is this task currently executing on a CPU? + * @p: the task in question. + */ +static int sc_task_curr(const task_t *p) +{ + return cpu_curr(task_cpu(p)) == p; +} + +#ifdef CONFIG_SMP +enum request_type { + REQ_MOVE_TASK, + REQ_SET_DOMAIN, +}; + +typedef struct { + struct list_head list; + enum request_type type; + + /* For REQ_MOVE_TASK */ + task_t *task; + int dest_cpu; + + /* For REQ_SET_DOMAIN */ + struct sched_domain *sd; + + struct completion done; +} migration_req_t; + +/* + * The task's runqueue lock must be held. + * Returns true if you have to wait for migration thread. + */ +static int migrate_task(task_t *p, int dest_cpu, migration_req_t *req) +{ + runqueue_t *rq = task_rq(p); + + /* + * If the task is not on a runqueue (and not running), then + * it is sufficient to simply update the task's cpu field. + */ + if (!task_queued(p) && !task_running(rq, p)) { + set_task_cpu(p, dest_cpu); + return 0; + } + + init_completion(&req->done); + req->type = REQ_MOVE_TASK; + req->task = p; + req->dest_cpu = dest_cpu; + list_add(&req->list, &rq->migration_queue); + return 1; +} + +/* + * wait_task_inactive - wait for a thread to unschedule. + * + * The caller must ensure that the task *will* unschedule sometime soon, + * else this function might spin for a *long* time. This function can't + * be called with interrupts off, or it may introduce deadlock with + * smp_call_function() if an IPI is sent by the same process we are + * waiting to become inactive. + */ +static void sc_wait_task_inactive(task_t * p) +{ + unsigned long flags; + runqueue_t *rq; + int preempted; + +repeat: + rq = task_rq_lock(p, &flags); + /* Must be off runqueue entirely, not preempted. */ + if (unlikely(task_queued(p))) { + /* If it's preempted, we yield. It could be a while. */ + preempted = !task_running(rq, p); + task_rq_unlock(rq, &flags); + cpu_relax(); + if (preempted) + yield(); + goto repeat; + } + task_rq_unlock(rq, &flags); +} + +/* + * Return a low guess at the load of a migration-source cpu. + * + * We want to under-estimate the load of migration sources, to + * balance conservatively. + */ +static inline unsigned long source_load(int cpu) +{ + runqueue_t *rq = cpu_rq(cpu); + unsigned long load_now = rq->nr_running * SCHED_LOAD_SCALE; + + return min(rq->cpu_load, load_now); +} + +/* + * Return a high guess at the load of a migration-target cpu + */ +static inline unsigned long target_load(int cpu) +{ + runqueue_t *rq = cpu_rq(cpu); + unsigned long load_now = rq->nr_running * SCHED_LOAD_SCALE; + + return max(rq->cpu_load, load_now); +} + +#endif + +/* + * wake_idle() will wake a task on an idle cpu if task->cpu is + * not idle and an idle cpu is available. The span of cpus to + * search starts with cpus closest then further out as needed, + * so we always favor a closer, idle cpu. + * + * Returns the CPU we should wake onto. + */ +#if defined(ARCH_HAS_SCHED_W