Force binding of tasks to CPU0 when one of its CPUs that it's affined to is
going offline, setting a zerobound flag which is only unset once all the CPUs
are back online. Treat kernel threads separately since they disappear on
suspend and any new ones created before a CPU comes up will have their own
special affinity settings.

-ck
---
 include/linux/sched.h |    3 ++
 kernel/sched/bfs.c    |   62 ++++++++++++++++++++++++++++++++++++++++++--------
 2 files changed, 56 insertions(+), 9 deletions(-)

Index: linux-3.12-ck1/include/linux/sched.h
===================================================================
--- linux-3.12-ck1.orig/include/linux/sched.h	2013-12-02 09:56:58.000000000 +1100
+++ linux-3.12-ck1/include/linux/sched.h	2013-12-02 14:25:43.532858113 +1100
@@ -1045,6 +1045,9 @@ struct task_struct {
 #ifdef CONFIG_SMP
 	bool sticky; /* Soft affined flag */
 #endif
+#ifdef CONFIG_HOTPLUG_CPU
+	bool zerobound; /* Bound to CPU0 for hotplug */
+#endif
 	unsigned long rt_timeout;
 #else /* CONFIG_SCHED_BFS */
 	const struct sched_class *sched_class;
Index: linux-3.12-ck1/kernel/sched/bfs.c
===================================================================
--- linux-3.12-ck1.orig/kernel/sched/bfs.c	2013-12-02 10:39:35.133023202 +1100
+++ linux-3.12-ck1/kernel/sched/bfs.c	2013-12-02 14:42:31.596524623 +1100
@@ -5385,26 +5385,66 @@ EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
 
 #ifdef CONFIG_HOTPLUG_CPU
 extern struct task_struct *cpu_stopper_task;
-/* Run through task list and find tasks affined to just the dead cpu, then
- * allocate a new affinity */
-static void break_sole_affinity(int src_cpu, struct task_struct *idle)
+/* Run through task list and find tasks affined to the dead cpu, then remove
+ * that cpu from the list, enable cpu0 and set the zerobound flag. */
+static void bind_zero(int src_cpu)
 {
 	struct task_struct *p, *t, *stopper;
-	int unbound = 0;
+	int bound = 0;
+
+	if (src_cpu == 0)
+		return;
 
 	stopper = per_cpu(cpu_stopper_task, src_cpu);
 	do_each_thread(t, p) {
-		if (p != stopper && !online_cpus(p)) {
-			cpuset_cpus_allowed_fallback(p);
-			unbound++;
+		if (p != stopper && cpu_isset(src_cpu, *tsk_cpus_allowed(p))) {
+			cpumask_clear_cpu(src_cpu, tsk_cpus_allowed(p));
+			cpumask_set_cpu(0, tsk_cpus_allowed(p));
+			p->zerobound = true;
+			bound++;
 		}
 		clear_sticky(p);
 	} while_each_thread(t, p);
 
+	if (bound) {
+		printk(KERN_INFO "Removed affinity for %d processes to cpu %d\n",
+		       bound, src_cpu);
+	}
+}
+
+/* Find processes with the zerobound flag and reenable their affinity for the
+ * CPU coming alive. */
+static void unbind_zero(int src_cpu)
+{
+	int unbound = 0, zerobound = 0;
+	struct task_struct *p, *t;
+
+	if (src_cpu == 0)
+		return;
+
+	do_each_thread(t, p) {
+		if (!p->mm)
+			p->zerobound = false;
+		if (p->zerobound) {
+			unbound++;
+			cpumask_set_cpu(src_cpu, tsk_cpus_allowed(p));
+			/* Once every CPU affinity has been re-enabled, remove
+			 * the zerobound flag */
+			if (cpumask_subset(cpu_possible_mask, tsk_cpus_allowed(p))) {
+				p->zerobound = false;
+				zerobound++;
+			}
+		}
+	} while_each_thread(t, p);
+
 	if (unbound) {
-		printk(KERN_INFO "Broke affinity for %d processes to cpu %d\n",
+		printk(KERN_INFO "Added affinity for %d processes to cpu %d\n",
 		       unbound, src_cpu);
 	}
+	if (zerobound) {
+		printk(KERN_INFO "Released forced binding to cpu0 for %d processes\n",
+		       zerobound);
+	}
 }
 
 /*
@@ -5421,7 +5461,10 @@ void idle_task_exit(void)
 		switch_mm(mm, &init_mm, current);
 	mmdrop(mm);
 }
+#else /* CONFIG_HOTPLUG_CPU */
+static void unbind_zero(int src_cpu) {}
 #endif /* CONFIG_HOTPLUG_CPU */
+
 void sched_set_stop_task(int cpu, struct task_struct *stop)
 {
 	struct sched_param stop_param = { .sched_priority = STOP_PRIO };
@@ -5660,6 +5703,7 @@ migration_call(struct notifier_block *nf
 
 			set_rq_online(rq);
 		}
+		unbind_zero(cpu);
 		grq.noc = num_online_cpus();
 		grq_unlock_irqrestore(&flags);
 		break;
@@ -5680,7 +5724,7 @@ migration_call(struct notifier_block *nf
 			BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
 			set_rq_offline(rq);
 		}
-		break_sole_affinity(cpu, idle);
+		bind_zero(cpu);
 		grq.noc = num_online_cpus();
 		grq_unlock_irqrestore(&flags);
 		break;