Index: linux-2.6.11-ISO/kernel/sched.c =================================================================== --- linux-2.6.11-ISO.orig/kernel/sched.c 2005-04-16 11:10:23.000000000 +1000 +++ linux-2.6.11-ISO/kernel/sched.c 2005-04-16 19:06:17.000000000 +1000 @@ -318,14 +318,15 @@ static int task_preempts_curr(task_t *p, if (!iso_task(p) && !iso_task(rq->curr)) goto out; - if (!rq->iso_refractory) { + if (likely(!rq->iso_refractory)) { if (iso_task(p)) { if (iso_task(rq->curr)) { - p_prio = -p->rt_priority; - curr_prio = -rq->curr->rt_priority; + p_prio = -(p->rt_priority); + curr_prio = -(rq->curr->rt_priority); goto out; } p_prio = ISO_PRIO; + goto out; } if (iso_task(rq->curr)) curr_prio = ISO_PRIO; @@ -610,6 +611,14 @@ static void dequeue_iso_task(struct task __clear_bit(iso_prio(p), rq->iso_bitmap); } +static inline void __dequeue_task(struct task_struct *p, prio_array_t *array) +{ + array->nr_active--; + list_del(&p->run_list); + if (list_empty(array->queue + p->prio)) + __clear_bit(p->prio, array->bitmap); +} + /* * Adding/removing a task to/from a priority array: */ @@ -617,10 +626,7 @@ static void dequeue_task(struct task_str { if (iso_task(p)) dequeue_iso_task(p, rq); - array->nr_active--; - list_del(&p->run_list); - if (list_empty(array->queue + p->prio)) - __clear_bit(p->prio, array->bitmap); + __dequeue_task(p, array); } /* @@ -635,13 +641,18 @@ static void enqueue_iso_task(struct task rq->iso_running++; } -static void enqueue_task(struct task_struct *p, runqueue_t *rq, prio_array_t *array) +static inline void __enqueue_task(struct task_struct *p, prio_array_t *array) { sched_info_queued(p); list_add_tail(&p->run_list, array->queue + p->prio); __set_bit(p->prio, array->bitmap); array->nr_active++; p->array = array; +} + +static void enqueue_task(struct task_struct *p, runqueue_t *rq, prio_array_t *array) +{ + __enqueue_task(p, array); if (iso_task(p)) enqueue_iso_task(p, rq); } @@ -2478,27 +2489,23 @@ void account_steal_time(struct task_stru cpustat->steal = cputime64_add(cpustat->steal, tmp); } -static inline void inc_iso_ticks(runqueue_t *rq, task_t *p) +static inline void inc_iso_ticks(runqueue_t *rq) { if (rq->iso_ticks < (iso_period * HZ * 100 - 99)) rq->iso_ticks += 100; - spin_lock(&rq->lock); - if (!rq->iso_refractory && (rq->iso_ticks / - ((iso_period * HZ) + 1) > iso_cpu)) + if (unlikely(!rq->iso_refractory && (rq->iso_ticks / + ((iso_period * HZ) + 1) > iso_cpu))) rq->iso_refractory = 1; - spin_unlock(&rq->lock); } -static inline void dec_iso_ticks(runqueue_t *rq, task_t *p) +static inline void dec_iso_ticks(runqueue_t *rq) { if (rq->iso_ticks) rq->iso_ticks = rq->iso_ticks * (iso_period * HZ - 1) / (iso_period * HZ); - spin_lock(&rq->lock); - if (rq->iso_refractory && (rq->iso_ticks / - ((iso_period * HZ) + 1) < (iso_cpu * 9 / 10))) + if (unlikely(rq->iso_refractory && (rq->iso_ticks / + ((iso_period * HZ) + 1) < (iso_cpu * 9 / 10)))) rq->iso_refractory = 0; - spin_unlock(&rq->lock); } /* @@ -2516,18 +2523,8 @@ void scheduler_tick(void) rq->timestamp_last_tick = sched_clock(); - /* - * The iso_ticks accounting is incremented only when a SCHED_ISO task - * is running in soft rt mode. Running rt_tasks are also accounted - * to make the iso_cpu a proportion of cpu available for SCHED_NORMAL - * tasks only. - */ - if (rt_task(p) || (iso_task(p) && !rq->iso_refractory)) - inc_iso_ticks(rq, p); - else - dec_iso_ticks(rq, p); - if (p == rq->idle) { + dec_iso_ticks(rq); if (wake_priority_sleeper(rq)) goto out; rebalance_tick(cpu, rq, SCHED_IDLE); @@ -2549,37 +2546,50 @@ void scheduler_tick(void) * to use up their timeslices at their highest priority levels. */ if (rt_task(p)) { - /* - * RR tasks need a special form of timeslice management. - * FIFO tasks have no timeslices. - */ - if ((p->policy == SCHED_RR) && !--p->time_slice) { - p->time_slice = task_timeslice(p); - p->first_time_slice = 0; - set_tsk_need_resched(p); - - /* put it at the end of the queue: */ - requeue_task(p, rq, rq->active); - } - goto out_unlock; + if (p->policy == SCHED_RR) + goto sched_rr; + goto sched_fifo; } - if (iso_task(p)) { - if (rq->iso_refractory) + if (unlikely(rq->iso_refractory)) /* * If we are in the refractory period for SCHED_ISO - * tasks we schedule them as SCHED_NORMAL. + * tasks we schedule them as SCHED_NORMAL. Otherwise + * schedule them as their rt equivalents. */ goto sched_normal; - if (p->policy == SCHED_ISO_RR && !--p->time_slice) { - p->time_slice = task_timeslice(p); - set_tsk_need_resched(p); - requeue_iso_task(p, rq); - } - goto out_unlock; + if (p->policy == SCHED_ISO_RR) + goto sched_rr; + goto sched_fifo; + } + goto sched_normal; + +sched_rr: + /* + * RR tasks need a special form of timeslice management. + */ + if (!--p->time_slice) { + p->time_slice = task_timeslice(p); + p->first_time_slice = 0; + set_tsk_need_resched(p); + + /* put it at the end of the queue: */ + requeue_task(p, rq, rq->active); } +sched_fifo: + /* + * The iso_ticks accounting is incremented only when a SCHED_ISO task + * is running in soft rt mode. Running rt_tasks are also accounted + * to make the iso_cpu a proportion of cpu available for SCHED_NORMAL + * tasks only. + */ + inc_iso_ticks(rq); + /* FIFO tasks have no timeslices. */ + goto out_unlock; + sched_normal: + dec_iso_ticks(rq); if (!--p->time_slice) { dequeue_task(p, rq, rq->active); set_tsk_need_resched(p); @@ -2648,15 +2658,9 @@ static task_t* find_iso(runqueue_t *rq, if (p->array == array) goto out; old_array = p->array; - old_array->nr_active--; - list_del(&p->run_list); - if (list_empty(old_array->queue + p->prio)) - __clear_bit(p->prio, old_array->bitmap); - list_add_tail(&p->run_list, array->queue + p->prio); - __set_bit(p->prio, array->bitmap); - array->nr_active++; - p->array = array; -out: + __dequeue_task(p, old_array); + __enqueue_task(p, array); +out: return p; } @@ -3571,6 +3575,7 @@ int sched_setscheduler(struct task_struc { int retval; int oldprio, oldpolicy = -1; + struct sched_param zero_param = { .sched_priority = 0 }; prio_array_t *array; unsigned long flags; runqueue_t *rq; @@ -3586,7 +3591,8 @@ recheck: policy = SCHED_ISO_RR; else policy = SCHED_ISO_FIFO; - } + param = &zero_param; + } /* double check policy once rq lock held */ if (policy < 0)