Index: linux-2.6.11-smpnice/kernel/sched.c =================================================================== --- linux-2.6.11-smpnice.orig/kernel/sched.c 2005-03-02 18:38:19.000000000 +1100 +++ linux-2.6.11-smpnice/kernel/sched.c 2005-06-02 10:03:38.000000000 +1000 @@ -204,6 +204,7 @@ struct runqueue { */ unsigned long nr_running; #ifdef CONFIG_SMP + unsigned long prio_bias; unsigned long cpu_load; #endif unsigned long long nr_switches; @@ -628,13 +629,51 @@ static int effective_prio(task_t *p) return prio; } +#ifdef CONFIG_SMP +static inline void inc_prio_bias(runqueue_t *rq, int prio) +{ + rq->prio_bias += MAX_PRIO - prio; +} + +static inline void dec_prio_bias(runqueue_t *rq, int prio) +{ + rq->prio_bias -= MAX_PRIO - prio; +} +#else +static inline void inc_prio_bias(runqueue_t *rq, int prio) +{ +} + +static inline void dec_prio_bias(runqueue_t *rq, int prio) +{ +} +#endif + +static inline void inc_nr_running(task_t *p, runqueue_t *rq) +{ + rq->nr_running++; + if (rt_task(p)) + inc_prio_bias(rq, p->prio); + else + inc_prio_bias(rq, p->static_prio); +} + +static inline void dec_nr_running(task_t *p, runqueue_t *rq) +{ + rq->nr_running--; + if (rt_task(p)) + dec_prio_bias(rq, p->prio); + else + dec_prio_bias(rq, p->static_prio); +} + /* * __activate_task - move a task to the runqueue. */ static inline void __activate_task(task_t *p, runqueue_t *rq) { enqueue_task(p, rq->active); - rq->nr_running++; + inc_nr_running(p, rq); } /* @@ -643,7 +682,7 @@ static inline void __activate_task(task_ static inline void __activate_idle_task(task_t *p, runqueue_t *rq) { enqueue_task_head(p, rq->active); - rq->nr_running++; + inc_nr_running(p, rq); } static void recalc_task_prio(task_t *p, unsigned long long now) @@ -761,7 +800,7 @@ static void activate_task(task_t *p, run */ static void deactivate_task(struct task_struct *p, runqueue_t *rq) { - rq->nr_running--; + dec_nr_running(p, rq); dequeue_task(p, p->array); p->array = NULL; } @@ -909,25 +948,52 @@ void kick_process(task_t *p) * We want to under-estimate the load of migration sources, to * balance conservatively. */ -static inline unsigned long source_load(int cpu) +static inline unsigned long __source_load(int cpu, enum idle_type idle) { runqueue_t *rq = cpu_rq(cpu); - unsigned long load_now = rq->nr_running * SCHED_LOAD_SCALE; + unsigned long source_load, cpu_load = rq->cpu_load, + load_now = rq->nr_running * SCHED_LOAD_SCALE; + + source_load = min(cpu_load, load_now); + if (idle == NOT_IDLE || rq->nr_running > 1) + /* + * If we are busy rebalancing the load is biased by + * priority to create 'nice' support across cpus. When + * idle rebalancing we should only bias the source_load if + * there is more than one task running on that queue to + * prevent idle rebalance from trying to pull tasks from a + * queue with only one running task. + */ + source_load *= rq->prio_bias; + + return source_load; +} - return min(rq->cpu_load, load_now); +static inline unsigned long source_load(int cpu) +{ + return __source_load(cpu, NOT_IDLE); } /* * Return a high guess at the load of a migration-target cpu */ -static inline unsigned long target_load(int cpu) +static inline unsigned long __target_load(int cpu, enum idle_type idle) { runqueue_t *rq = cpu_rq(cpu); - unsigned long load_now = rq->nr_running * SCHED_LOAD_SCALE; + unsigned long target_load, cpu_load = rq->cpu_load, + load_now = rq->nr_running * SCHED_LOAD_SCALE; + + target_load = max(cpu_load, load_now); + if (idle == NOT_IDLE || rq->nr_running > 1) + target_load *= rq->prio_bias; - return max(rq->cpu_load, load_now); + return target_load; } +static inline unsigned long target_load(int cpu) +{ + return __target_load(cpu, NOT_IDLE); +} #endif /* @@ -1240,7 +1306,7 @@ void fastcall wake_up_new_task(task_t * list_add_tail(&p->run_list, ¤t->run_list); p->array = current->array; p->array->nr_active++; - rq->nr_running++; + inc_nr_running(p, rq); } set_need_resched(); } else @@ -1630,9 +1696,9 @@ void pull_task(runqueue_t *src_rq, prio_ runqueue_t *this_rq, prio_array_t *this_array, int this_cpu) { dequeue_task(p, src_array); - src_rq->nr_running--; + dec_nr_running(p, src_rq); set_task_cpu(p, this_cpu); - this_rq->nr_running++; + inc_nr_running(p, this_rq); enqueue_task(p, this_array); p->timestamp = (p->timestamp - src_rq->timestamp_last_tick) + this_rq->timestamp_last_tick; @@ -1790,9 +1856,9 @@ find_busiest_group(struct sched_domain * for_each_cpu_mask(i, group->cpumask) { /* Bias balancing toward cpus of our domain */ if (local_group) - load = target_load(i); + load = __target_load(i, idle); else - load = source_load(i); + load = __source_load(i, idle); nr_cpus++; avg_load += load; @@ -1904,14 +1970,15 @@ out_balanced: /* * find_busiest_queue - find the busiest runqueue among the cpus in group. */ -static runqueue_t *find_busiest_queue(struct sched_group *group) +static runqueue_t *find_busiest_queue(struct sched_group *group, + enum idle_type idle) { unsigned long load, max_load = 0; runqueue_t *busiest = NULL; int i; for_each_cpu_mask(i, group->cpumask) { - load = source_load(i); + load = __source_load(i, idle); if (load > max_load) { max_load = load; @@ -1945,7 +2012,7 @@ static int load_balance(int this_cpu, ru goto out_balanced; } - busiest = find_busiest_queue(group); + busiest = find_busiest_queue(group, idle); if (!busiest) { schedstat_inc(sd, lb_nobusyq[idle]); goto out_balanced; @@ -2049,7 +2116,7 @@ static int load_balance_newidle(int this goto out; } - busiest = find_busiest_queue(group); + busiest = find_busiest_queue(group, NEWLY_IDLE); if (!busiest || busiest == this_rq) { schedstat_inc(sd, lb_nobusyq[NEWLY_IDLE]); goto out; @@ -3249,8 +3316,10 @@ void set_user_nice(task_t *p, long nice) goto out_unlock; } array = p->array; - if (array) + if (array) { dequeue_task(p, array); + dec_prio_bias(rq, p->static_prio); + } old_prio = p->prio; new_prio = NICE_TO_PRIO(nice); @@ -3260,6 +3329,7 @@ void set_user_nice(task_t *p, long nice) if (array) { enqueue_task(p, array); + inc_prio_bias(rq, p->static_prio); /* * If the task increased its priority or is running and * lowered its priority, then reschedule its CPU: