Index: linux-2.6.15-rc6-ck1/arch/i386/kernel/dyn-tick.c =================================================================== --- linux-2.6.15-rc6-ck1.orig/arch/i386/kernel/dyn-tick.c +++ linux-2.6.15-rc6-ck1/arch/i386/kernel/dyn-tick.c @@ -23,7 +23,6 @@ #include #include -/* We only skip 200ms maximum so no need to disable the NMI Watchdog */ static void apic_reprogram(unsigned long jif_next) { reprogram_apic_timer(jif_next - jiffies); @@ -34,44 +33,81 @@ static void pit_reprogram(unsigned long reprogram_pit_timer(jif_next - jiffies); } -static void arch_all_cpus_idle(int __unused) +/* + * This is a special case that does pit reprogramming only when all cpus are + * idle on an APIC SMP system that does not have reliable APIC reprogramming + * with the PIT disabled. + */ +static void reprogram_pit_handler(void) { - if (dyntick_using_lapic()) - disable_pit_timer(); + unsigned long skip = dyn_tick->tick - jiffies; + + if (skip > PIT_MAX_SKIP) + skip = PIT_MAX_SKIP; + reprogram_pit_timer(skip); } -static int arch_enable(void) +/* + * This reprograms the pit to maximum skips when calibrating APIC reprogramming + * As it will often delay the pit longer than APIC is skipping it is a good way + * to see if APIC timers work on their own. + */ +static void reprogram_max_pit(void) { - return 0; + reprogram_pit_timer(PIT_MAX_SKIP); } -static int arch_disable(void) +static inline int null_enable(void) { return 0; } +static inline void null_pit_handler(void) +{ +} + +static void do_dyn_tick_interrupt(struct pt_regs *regs); + +struct arch_dyn_handler arch_handler = { + .idle_pit_handler = &null_pit_handler, + .busy_pit_handler = &null_pit_handler, + .do_dyn_interrupt = &do_dyn_tick_interrupt, + .pit_handler = PIT_UP_SKIP, +}; + +static void arch_all_cpus_idle(int __unused) +{ + arch_handler.idle_pit_handler(); +} + static struct dyn_tick_timer arch_dyn_tick = { .lock = SPIN_LOCK_UNLOCKED, .arch_reprogram = &pit_reprogram, .arch_all_cpus_idle = &arch_all_cpus_idle, - .arch_enable = &arch_enable, - .arch_disable = &arch_disable, + .arch_enable = &null_enable, + .arch_disable = &null_enable, }; struct dyn_tick_timer *dyn_tick = &arch_dyn_tick; +static inline void show_limits(void) +{ + printk(KERN_INFO "dyn-tick: Maximum ticks to skip limited to %i\n", + dyn_tick->max_skip); +} + static inline void set_pit_limits(void) { - set_dyn_tick_limits(0xffff / LATCH, 0); + set_dyn_tick_limits(PIT_MAX_SKIP, 0); } int __init dyn_tick_arch_init(void) { - if (!dyntick_using_lapic()) + if (arch_handler.pit_handler == PIT_UP_SKIP) { set_pit_limits(); - - printk(KERN_INFO "dyn-tick: Maximum ticks to skip limited to %i\n", - dyn_tick->max_skip); + printk(KERN_INFO "dyn-tick: Using PIT reprogramming\n"); + show_limits(); + } return 0; } @@ -86,62 +122,176 @@ static int __init dyn_tick_init(void) arch_initcall(dyn_tick_init); -static void dyntick_disable_lapic(void) +/* + * The apparently redundant per_cpu nohz_cpu value is tested in this + * function and this is where we can avoid the cache thrashing of testing + * nohz_cpu_mask. + */ +static inline int test_nohz_cpu(void) +{ + return __get_cpu_var(dyn_cpu).nohz_cpu; +} + +static inline void clear_nohz_cpu(int cpu) { - dyn_tick->state &= ~DYN_TICK_APICABLE; - printk(KERN_INFO "dyn-tick: Cannot use this local APIC with dynticks\n"); - set_pit_limits(); - dyn_tick->arch_reprogram = &pit_reprogram; + __get_cpu_var(dyn_cpu).nohz_cpu = 0; + cpu_clear(cpu, nohz_cpu_mask); } -/* Functions that need blank prototypes for !CONFIG_NO_IDLE_HZ below here */ -int __init setup_dyn_tick_use_apic(unsigned int calibration_result) +static void do_dyn_tick_interrupt(struct pt_regs *regs) { - if (calibration_result) { - dyn_tick->arch_reprogram = &apic_reprogram; - dyn_tick->state |= DYN_TICK_APICABLE; - return 1; + int cpu = smp_processor_id(); + + write_seqlock(&xtime_lock); + spin_lock(&dyn_tick->lock); + if (cpus_equal(nohz_cpu_mask, cpu_online_map)) { + /* All were sleeping, recover jiffies */ + int lost = cur_timer->mark_offset(); + + if (lost && in_irq()) + do_timer(regs); + arch_handler.busy_pit_handler(); + kstat_cpu(cpu).cpustat.idle += (lost - 1); } - dyntick_disable_lapic(); - return 0; + clear_nohz_cpu(cpu); + spin_unlock(&dyn_tick->lock); + write_sequnlock(&xtime_lock); + + /* Make sure we don't miss the next timer tick */ + dyn_early_reprogram(1); + + conditional_run_local_timers(); +} + +static void select_apic_handler(void) +{ + switch (arch_handler.pit_handler) { + case PIT_SKIP: + printk(KERN_INFO + "dyn-tick: Using APIC reprogramming, skipping PIT when all cpus idle\n"); + break; + case PIT_UP_SKIP: + printk(KERN_INFO + "dyn-tick: local APIC unreliable, using PIT reprogramming\n"); + /* + * Reprogram the APIC timer to run quietly at the slowest rate + * possible to not upset the NMI watchdog. + */ + apic_reprogram(dyn_tick->max_skip); + set_pit_limits(); + break; + case PIT_DISABLE: + printk(KERN_INFO + "dyn-tick: Using APIC reprogramming, disabling PIT when all cpus idle\n"); + break; + case PIT_OFF: + case PIT_UP_APIC: + printk(KERN_INFO + "dyn-tick: Using APIC reprogramming, turning PIT off entirely\n"); + disable_pit_timer(); + break; + } + arch_handler.do_dyn_interrupt = &do_dyn_tick_interrupt; + show_limits(); } /* - * The apparently redundant per_cpu nohz_cpu value is tested in this - * function and this is where we can avoid the cache thrashing of testing - * nohz_cpu_mask. Preemption is already disabled when this is called. + * On SMP if there is a delay with PIT off for extended periods we can drop + * to disabling the pit timer only when all cpus are idle. If that fails we + * must move to pit reprogramming when all cpus are idle. + * + * On UP we either have PIT off with APIC reprogramming or use PIT + * reprogramming. Most only do the latter reliably. */ -static inline int test_nohz_cpu(void) +static void select_handler(void) { - return __get_cpu_var(nohz_cpu); + switch (arch_handler.pit_handler) { + case PIT_SKIP: + dyn_tick->arch_reprogram = &apic_reprogram; + arch_handler.idle_pit_handler = &reprogram_pit_handler; + arch_handler.busy_pit_handler = &null_pit_handler; + break; + case PIT_UP_SKIP: + dyn_tick->arch_reprogram = &pit_reprogram; + arch_handler.idle_pit_handler = &null_pit_handler; + arch_handler.busy_pit_handler = &null_pit_handler; + select_apic_handler(); + break; + case PIT_DISABLE: + dyn_tick->arch_reprogram = &apic_reprogram; + arch_handler.idle_pit_handler = &disable_pit_timer; + arch_handler.busy_pit_handler = &enable_pit_timer; + break; + case PIT_OFF: + case PIT_UP_APIC: + dyn_tick->arch_reprogram = &apic_reprogram; + arch_handler.idle_pit_handler = &null_pit_handler; + arch_handler.busy_pit_handler = &null_pit_handler; + break; + } } -static inline void clear_nohz_cpu(int cpu) +/* + * Select the next most suitable handler based on the previous handler. + */ +static void failed_pit_handler(void) { - __get_cpu_var(nohz_cpu) = 0; - cpu_clear(cpu, nohz_cpu_mask); + switch (arch_handler.pit_handler) { + case PIT_SKIP: + case PIT_UP_SKIP: + return; + case PIT_DISABLE: + arch_handler.pit_handler = PIT_SKIP; + break; + case PIT_OFF: + arch_handler.pit_handler = PIT_DISABLE; + break; + case PIT_UP_APIC: + arch_handler.pit_handler = PIT_UP_SKIP; + break; + } + select_handler(); } -static void do_dyn_tick_interrupt(struct pt_regs *regs) +/* + * This function is used instead of do_dyn_tick_interrupt to determine what + * we need to do with the pit timer. We may be able to disable it completely, + * disable it only when all cpus are idle or have to drop APIC reprogramming + * entirely. + */ +static void apic_calibrate_interrupt(struct pt_regs *regs) { + static int count = 0; int cpu = smp_processor_id(); write_seqlock(&xtime_lock); spin_lock(&dyn_tick->lock); + count++; + if (cpus_equal(nohz_cpu_mask, cpu_online_map)) { /* All were sleeping, recover jiffies */ int lost = cur_timer->mark_offset(); if (lost && in_irq()) do_timer(regs); - if (dyntick_using_lapic()) { - enable_pit_timer(); - if (unlikely(lost > DYN_TICK_MAX_SKIP + 1)) - dyntick_disable_lapic(); - } + arch_handler.busy_pit_handler(); + /* + * Here we test to see that we skipped only as much as APIC + * reprogramming specified and if we skipped more it displays + * failure without PIT - very common. + */ + if (lost > __get_cpu_var(dyn_cpu).skip + 2) + failed_pit_handler(); kstat_cpu(cpu).cpustat.idle += (lost - 1); } clear_nohz_cpu(cpu); + + /* + * We let this run till 1000 hardware interrupts are generated while + * the cpu is idle to get a reasonable picture of APIC performance + */ + if (count > 1000) + select_apic_handler(); spin_unlock(&dyn_tick->lock); write_sequnlock(&xtime_lock); @@ -151,11 +301,22 @@ static void do_dyn_tick_interrupt(struct conditional_run_local_timers(); } +void __init setup_dyn_tick_use_apic(unsigned int calibration_result) +{ + if (calibration_result) { + arch_handler.pit_handler = select_pit_handler(); + select_handler(); + arch_handler.idle_pit_handler = &reprogram_max_pit; + arch_handler.do_dyn_interrupt = &apic_calibrate_interrupt; + return; + } +} + inline void dyn_tick_interrupt(struct pt_regs *regs) { if (!test_nohz_cpu()) return; - do_dyn_tick_interrupt(regs); + arch_handler.do_dyn_interrupt(regs); } void __init dyn_tick_time_init(struct timer_opts *cur_timer) @@ -165,7 +326,8 @@ void __init dyn_tick_time_init(struct ti printk(KERN_INFO "dyn-tick: Found suitable timer: %s\n", cur_timer->name); } else - printk(KERN_ERR "dyn-tick: Cannot use timer %s\n", + printk(KERN_ERR + "dyn-tick: Cannot use timer %s - pmtmr failed: ACPI disabled?\n", cur_timer->name); } Index: linux-2.6.15-rc6-ck1/include/linux/dyn-tick.h =================================================================== --- linux-2.6.15-rc6-ck1.orig/include/linux/dyn-tick.h +++ linux-2.6.15-rc6-ck1/include/linux/dyn-tick.h @@ -20,8 +20,8 @@ #define DYN_TICK_ENABLED (1 << 1) #define DYN_TICK_SUITABLE (1 << 0) -/* Baseline tick rate of slowest timers in kernel - used for sanity checks */ -#define DYN_TICK_MAX_SKIP (HZ / 5) +/* Don't skip longer than NMI */ +#define DYN_TICK_MAX_SKIP (HZ * 5) struct dyn_tick_timer { spinlock_t lock; @@ -40,9 +40,14 @@ struct dyn_tick_timer { unsigned short state; /* Current state */ unsigned int min_skip; /* Min number of ticks to skip */ unsigned int max_skip; /* Max number of ticks to skip */ + unsigned long tick; /* The next earliest tick */ +}; + +typedef struct { unsigned long next_tick; /* Next tick we're skipping to */ unsigned long skip; /* Ticks we're currently skipping */ -}; + unsigned int nohz_cpu; /* This cpu is idle */ +} ____cacheline_aligned dyn_tick_data; extern struct dyn_tick_timer *dyn_tick; extern spinlock_t *dyn_tick_lock; @@ -50,7 +55,8 @@ extern spinlock_t *dyn_tick_lock; extern void dyn_tick_register(struct dyn_tick_timer *new_timer); #ifdef CONFIG_NO_IDLE_HZ -DECLARE_PER_CPU(unsigned int, nohz_cpu); +DECLARE_PER_CPU(dyn_tick_data, dyn_cpu); +extern dyn_tick_data dyn_cpu; extern int dyn_tick_enabled(void); extern int dyn_tick_skipping(void); extern int dyn_tick_current_skip(void); Index: linux-2.6.15-rc6-ck1/kernel/dyn-tick.c =================================================================== --- linux-2.6.15-rc6-ck1.orig/kernel/dyn-tick.c +++ linux-2.6.15-rc6-ck1/kernel/dyn-tick.c @@ -28,6 +28,8 @@ #define DYN_TICK_VERSION "051219" +DEFINE_PER_CPU(dyn_tick_data, dyn_cpu); + inline int dyn_tick_enabled(void) { return !!(dyn_tick->state & DYN_TICK_ENABLED); @@ -35,12 +37,16 @@ inline int dyn_tick_enabled(void) EXPORT_SYMBOL(dyn_tick_enabled); + /* * Returns if we are currently skipping ticks. */ -inline int dyn_tick_skipping(void) +int dyn_tick_skipping(void) { - return (dyn_tick->next_tick > jiffies); + int ret = (get_cpu_var(dyn_cpu).next_tick > jiffies); + + put_cpu_var(dyn_cpu); + return ret; } EXPORT_SYMBOL(dyn_tick_skipping); @@ -52,8 +58,9 @@ int dyn_tick_current_skip(void) { int ret = 0; - if (dyn_tick_skipping()) - ret = dyn_tick->skip; + if (get_cpu_var(dyn_cpu).next_tick > jiffies) + ret = __get_cpu_var(dyn_cpu).skip; + put_cpu_var(dyn_cpu); return ret; } @@ -66,8 +73,9 @@ unsigned long dyn_tick_next_tick(void) { unsigned long next = 0; - if (dyn_tick_skipping()) - next = dyn_tick->next_tick; + if (get_cpu_var(dyn_cpu).next_tick > jiffies) + next = __get_cpu_var(dyn_cpu).next_tick; + put_cpu_var(dyn_cpu); return next; } @@ -85,29 +93,26 @@ static inline int dyn_tick_suitable(void */ static void do_dyn_reprogram(unsigned int delta) { - dyn_tick->next_tick = jiffies + delta; - dyn_tick->skip = delta; - dyn_tick->arch_reprogram(dyn_tick->next_tick); + unsigned long next = jiffies + delta; + + __get_cpu_var(dyn_cpu).next_tick = next; + __get_cpu_var(dyn_cpu).skip = delta; + dyn_tick->arch_reprogram(next); /* We have to update the idle_timestamp */ - __get_cpu_var(irq_stat).idle_timestamp = dyn_tick->next_tick; + __get_cpu_var(irq_stat).idle_timestamp = next; + if (dyn_tick->tick <= jiffies || dyn_tick->tick > next) + dyn_tick->tick = next; } /* - * Per cpu nohz information. This seems redundant but querying this first - * elsewhere can avoid the need to query nohz_cpu_mask and the subsequent - * cache thrashing. - */ -DEFINE_PER_CPU(unsigned int, nohz_cpu); - -/* * We will be doing this only from the idle loop so preemption is already * disabled allowing us to use the direct __get_cpu_var */ static inline void set_nohz_cpu(int cpu) { - if (__get_cpu_var(nohz_cpu)) + if (__get_cpu_var(dyn_cpu).nohz_cpu) return; - __get_cpu_var(nohz_cpu) = 1; + __get_cpu_var(dyn_cpu).nohz_cpu = 1; cpu_set(cpu, nohz_cpu_mask); } @@ -131,20 +136,20 @@ void timer_dyn_reprogram(void) write_seqlock(&xtime_lock); delta = next_timer_interrupt() - jiffies; - spin_lock(&dyn_tick->lock); if (delta > dyn_tick->max_skip) delta = dyn_tick->max_skip; if (delta > dyn_tick->min_skip) { int idle_time = 0; - if (dyn_tick->next_tick == jiffies) - idle_time = dyn_tick->skip; + if (__get_cpu_var(dyn_cpu).next_tick == jiffies) + idle_time = __get_cpu_var(dyn_cpu).skip; + spin_lock(&dyn_tick->lock); do_dyn_reprogram(delta); set_nohz_cpu(cpu); if (cpus_equal(nohz_cpu_mask, cpu_online_map)) dyn_tick->arch_all_cpus_idle(idle_time); + spin_unlock(&dyn_tick->lock); } - spin_unlock(&dyn_tick->lock); write_sequnlock(&xtime_lock); } @@ -159,14 +164,17 @@ void dyn_early_reprogram(unsigned int de { unsigned long flags, tick = jiffies + delta; - if (tick >= dyn_tick->next_tick && dyn_tick->next_tick > jiffies) - return; + if (tick >= get_cpu_var(dyn_cpu).next_tick && + __get_cpu_var(dyn_cpu).next_tick > jiffies) + goto put_out; write_seqlock_irqsave(&xtime_lock, flags); spin_lock(&dyn_tick->lock); do_dyn_reprogram(delta); spin_unlock(&dyn_tick->lock); write_sequnlock_irqrestore(&xtime_lock, flags); +put_out: + put_cpu_var(dyn_cpu); } EXPORT_SYMBOL(dyn_early_reprogram); Index: linux-2.6.15-rc6-ck1/arch/i386/kernel/apic.c =================================================================== --- linux-2.6.15-rc6-ck1.orig/arch/i386/kernel/apic.c +++ linux-2.6.15-rc6-ck1/arch/i386/kernel/apic.c @@ -935,7 +935,6 @@ void (*wait_timer_tick)(void) __devinitd #define APIC_DIVISOR 16 static u32 apic_timer_val __read_mostly; -#define APIC_TIMER_VAL ((apic_timer_val) / (HZ)) static void __setup_APIC_LVTT(unsigned int clocks) { @@ -955,9 +954,9 @@ static void __setup_APIC_LVTT(unsigned i & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE)) | APIC_TDR_DIV_16); - apic_timer_val = clocks * HZ / APIC_DIVISOR; + apic_timer_val = clocks / APIC_DIVISOR; - apic_write_around(APIC_TMICT, APIC_TIMER_VAL); + apic_write_around(APIC_TMICT, apic_timer_val); } static void __devinit setup_APIC_timer(unsigned int clocks) @@ -981,7 +980,7 @@ void reprogram_apic_timer(unsigned long { unsigned long flags; - count = count * apic_timer_val / HZ; + count = count * apic_timer_val; local_irq_save(flags); apic_write_around(APIC_TMICT, count); local_irq_restore(flags); @@ -1082,9 +1081,8 @@ void __init setup_boot_APIC_clock(void) */ setup_APIC_timer(calibration_result); - if (setup_dyn_tick_use_apic(calibration_result)) - set_dyn_tick_limits((0xFFFFFFFF / calibration_result) * - APIC_DIVISOR, 0); + setup_dyn_tick_use_apic(calibration_result); + set_dyn_tick_limits((0xFFFFFFFF / calibration_result) * APIC_DIVISOR, 0); local_irq_restore(flags); } Index: linux-2.6.15-rc6-ck1/include/asm-i386/dyn-tick.h =================================================================== --- linux-2.6.15-rc6-ck1.orig/include/asm-i386/dyn-tick.h +++ linux-2.6.15-rc6-ck1/include/asm-i386/dyn-tick.h @@ -20,28 +20,39 @@ #ifdef CONFIG_NO_IDLE_HZ extern void idle_reprogram_timer(void); extern void dyn_tick_interrupt(struct pt_regs *regs); -extern int __init setup_dyn_tick_use_apic(unsigned int calibration_result); +extern void __init setup_dyn_tick_use_apic(unsigned int calibration_result); extern void __init dyn_tick_time_init(struct timer_opts *cur_timer); -/* - * Needs to be different to the arch independant values specified in - * include/linux/dyn-tick.h to share the dyn_tick_timer.state struct - * member. - */ -#define DYN_TICK_APICABLE (1 << 3) +#define PIT_MAX_SKIP (0xffff / (LATCH)) -#if (defined(CONFIG_SMP) || defined(CONFIG_X86_UP_APIC)) -static inline int dyntick_using_lapic(void) +enum pit_handler { + PIT_SKIP, + PIT_UP_SKIP, + PIT_DISABLE, + PIT_OFF, + PIT_UP_APIC, +}; + +struct arch_dyn_handler { + void (*idle_pit_handler)(void); + void (*busy_pit_handler)(void); + void (*do_dyn_interrupt)(struct pt_regs *regs); + enum pit_handler pit_handler; +}; + +extern struct arch_dyn_handler arch_handler; + +#ifdef CONFIG_SMP +static inline enum pit_handler select_pit_handler(void) { - return (dyn_tick->state & DYN_TICK_APICABLE); + return PIT_OFF; } - -#else /* (defined(CONFIG_SMP) || defined(CONFIG_X86_UP_APIC)) */ -static inline int dyntick_using_lapic(void) +#else /* CONFIG_SMP */ +static inline enum pit_handler select_pit_handler(void) { - return 0; + return PIT_UP_APIC; } -#endif /* (defined(CONFIG_SMP) || defined(CONFIG_X86_UP_APIC)) */ +#endif #else /* CONFIG_NO_IDLE_HZ */ static inline void idle_reprogram_timer(void) @@ -52,7 +63,7 @@ static inline void dyn_tick_interrupt(st { } -static inline int setup_dyn_tick_use_apic(unsigned int calibration_result) +static inline void setup_dyn_tick_use_apic(unsigned int calibration_result) { return 0; } Index: linux-2.6.15-rc6-ck1/drivers/input/serio/i8042.h =================================================================== --- linux-2.6.15-rc6-ck1.orig/drivers/input/serio/i8042.h +++ linux-2.6.15-rc6-ck1/drivers/input/serio/i8042.h @@ -44,7 +44,7 @@ * polling. */ -#define I8042_POLL_PERIOD HZ/20 +#define I8042_POLL_PERIOD HZ/5 /* * Status register bits.