---
 kernel/sched/bfs.c |   85 ++++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 58 insertions(+), 27 deletions(-)

Index: linux-3.3-ck1/kernel/sched/bfs.c
===================================================================
--- linux-3.3-ck1.orig/kernel/sched/bfs.c	2012-04-03 18:20:52.000000000 +1000
+++ linux-3.3-ck1/kernel/sched/bfs.c	2012-04-03 19:23:40.662854816 +1000
@@ -389,6 +389,33 @@ static inline int cpu_of(struct rq *rq)
  * increased since it last updated niffies, minus any added niffies by other
  * CPUs.
  */
+
+static inline void grq_upgrade_rwlock(void);
+static inline void grq_drop_wlock(void);
+
+/* Version requiring ulock only */
+static inline void __update_clocks(struct rq *rq)
+{
+	s64 ndiff;
+	long jdiff;
+
+	update_rq_clock(rq);
+	ndiff = rq->clock - rq->old_clock;
+	/* old_clock is only updated when we are updating niffies */
+	rq->old_clock = rq->clock;
+	ndiff -= grq.niffies - rq->last_niffy;
+	jdiff = jiffies - grq.last_jiffy;
+	niffy_diff(&ndiff, jdiff);
+
+	grq_upgrade_rwlock();
+	grq.last_jiffy += jdiff;
+	grq.niffies += ndiff;
+	grq_drop_wlock();
+
+	rq->last_niffy = grq.niffies;
+}
+
+/* Version for when write lock is already held */
 static inline void update_clocks(struct rq *rq)
 {
 	s64 ndiff;
@@ -589,6 +616,13 @@ static inline void grq_upgrade_rwlock(vo
 	__grq_write_lock();
 }
 
+/* Drop just the write component of an upgradeable lock */
+static inline void grq_drop_wlock(void)
+	__releases(grq.rwlock)
+{
+	__grq_write_unlock();
+}
+
 /* Downgrade upgradeable lock to read lock */
 static inline void grq_downgrade_rwlock(void)
 	__acquires(grq.rwlock)
@@ -639,7 +673,7 @@ static inline struct rq
 }
 
 static inline struct rq
-*task_grq_lock(struct task_struct *p, unsigned long *flags)
+*task_grq_ulock(struct task_struct *p, unsigned long *flags)
 	__acquires(grq.lock)
 {
 	grq_ulock_irqsave(flags);
@@ -689,7 +723,7 @@ static inline void task_grq_wunlock(unsi
 	grq_wunlock_irqrestore(flags);
 }
 
-static inline void task_grq_unlock(unsigned long *flags)
+static inline void task_grq_uunlock(unsigned long *flags)
 	__releases(grq.lock)
 {
 	grq_uunlock_irqrestore(flags);
@@ -732,7 +766,7 @@ static inline struct rq *__task_grq_wloc
 	return task_rq(p);
 }
 
-static inline struct rq *__task_grq_lock(struct task_struct *p)
+static inline struct rq *__task_grq_ulock(struct task_struct *p)
 	__acquires(grq.lock)
 {
 	grq_ulock();
@@ -746,7 +780,7 @@ static inline void __task_grq_wunlock(vo
 	grq_wunlock();
 }
 
-static inline void __task_grq_unlock(void)
+static inline void __task_grq_uunlock(void)
 	__releases(grq.lock)
 {
 	grq_uunlock();
@@ -1216,7 +1250,7 @@ void set_task_cpu(struct task_struct *p,
 		perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, NULL, 0);
 
 	/*
-	 * After ->cpu is set up to a new value, task_grq_lock(p, ...) can be
+	 * After ->cpu is set up to a new value, task_grq_ulock(p, ...) can be
 	 * successfully executed on another CPU. We must ensure that updates of
 	 * per-task data have been completed by this moment.
 	 */
@@ -1745,7 +1779,7 @@ static bool try_to_wake_up(struct task_s
 	 * No need to do time_lock_grq as we only need to update the rq clock
 	 * if we activate the task
 	 */
-	rq = task_grq_lock(p, &flags);
+	rq = task_grq_ulock(p, &flags);
 	cpu = task_cpu(p);
 
 	/* state is a volatile long, どうして、分からない */
@@ -1767,7 +1801,7 @@ out_unlock:
 	if (rwupgrade)
 		task_grq_wunlock(&flags);
 	else
-		task_grq_unlock(&flags);
+		task_grq_uunlock(&flags);
 
 	ttwu_stat(p, cpu, wake_flags);
 
@@ -1945,11 +1979,12 @@ void wake_up_new_task(struct task_struct
 	unsigned long flags;
 	struct rq *rq;
 
-	rq = task_grq_wlock(p, &flags);
+	rq = task_grq_ulock(p, &flags);
 	p->state = TASK_RUNNING;
 	parent = p->parent;
 	/* Unnecessary but small chance that the parent changed CPU */
 	set_task_cpu(p, task_cpu(parent));
+	grq_upgrade_rwlock();
 	activate_task(p, rq);
 	trace_sched_wakeup_new(p, 1);
 	if (rq->curr == parent && !suitable_idle_cpus(p)) {
@@ -2689,7 +2724,7 @@ ts_account:
  * Return any ns on the sched_clock that have not yet been accounted in
  * @p in case that task is currently running.
  *
- * Called with task_grq_lock() held.
+ * Called with task_grq_ulock() held.
  */
 unsigned long long task_delta_exec(struct task_struct *p)
 {
@@ -2697,16 +2732,14 @@ unsigned long long task_delta_exec(struc
 	struct rq *rq;
 	u64 ns = 0;
 
-	rq = task_grq_lock(p, &flags);
+	rq = task_grq_ulock(p, &flags);
 	if (p == rq->curr) {
-		grq_upgrade_rwlock();
-		update_clocks(rq);
+		__update_clocks(rq);
 		ns = rq->clock_task - rq->rq_last_ran;
 		if (unlikely((s64)ns < 0))
 			ns = 0;
-		task_grq_wunlock(&flags);
-	} else
-		task_grq_unlock(&flags);
+	}
+	task_grq_uunlock(&flags);
 
 	return ns;
 }
@@ -2722,17 +2755,15 @@ unsigned long long task_sched_runtime(st
 	u64 ps, ns = 0;
 	struct rq *rq;
 
-	rq = task_grq_lock(p, &flags);
+	rq = task_grq_ulock(p, &flags);
 	ps = p->sched_time;
 	if (p == rq->curr) {
-		grq_upgrade_rwlock();
-		update_clocks(rq);
+		__update_clocks(rq);
 		ns = rq->clock_task - rq->rq_last_ran;
 		if (unlikely((s64)ns < 0))
 			ns = 0;
-		task_grq_wunlock(&flags);
-	} else
-		task_grq_unlock(&flags);
+	}
+	task_grq_uunlock(&flags);
 	ps += ns;
 
 	return ps;
@@ -4375,13 +4406,13 @@ recheck:
 	 * To be able to change p->policy safely, the grunqueue lock must be
 	 * held.
 	 */
-	rq = __task_grq_lock(p);
+	rq = __task_grq_ulock(p);
 
 	/*
 	 * Changing the policy of the stop threads its a very bad idea
 	 */
 	if (p == rq->stop) {
-		__task_grq_unlock();
+		__task_grq_uunlock();
 		raw_spin_unlock_irqrestore(&p->pi_lock, flags);
 		return -EINVAL;
 	}
@@ -4391,7 +4422,7 @@ recheck:
 	 */
 	if (unlikely(policy == p->policy && (!is_rt_policy(policy) ||
 			param->sched_priority == p->rt_priority))) {
-		__task_grq_unlock();
+		__task_grq_uunlock();
 		raw_spin_unlock_irqrestore(&p->pi_lock, flags);
 		return 0;
 	}
@@ -4399,7 +4430,7 @@ recheck:
 	/* recheck policy now with rq lock held */
 	if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
 		policy = oldpolicy = -1;
-		__task_grq_unlock();
+		__task_grq_uunlock();
 		raw_spin_unlock_irqrestore(&p->pi_lock, flags);
 		goto recheck;
 	}
@@ -5270,7 +5301,7 @@ int set_cpus_allowed_ptr(struct task_str
 	struct rq *rq;
 	int ret = 0;
 
-	rq = task_grq_lock(p, &flags);
+	rq = task_grq_ulock(p, &flags);
 
 	if (cpumask_equal(tsk_cpus_allowed(p), new_mask))
 		goto out;
@@ -5312,7 +5343,7 @@ out:
 	if (rwupgrade)
 		task_grq_wunlock(&flags);
 	else
-		task_grq_unlock(&flags);
+		task_grq_uunlock(&flags);
 
 	if (running_wrong)
 		_cond_resched();