diff options
author | Thomas Gleixner <tglx@linutronix.de> | 2012-10-28 13:46:16 +0000 |
---|---|---|
committer | Steven Rostedt <rostedt@rostedt.homelinux.com> | 2013-06-06 21:28:21 -0400 |
commit | 245b5b52899cc9a695441b1d905e66c12cbe1059 (patch) | |
tree | 2547f74736b968d7833fdd701b5a76a585e9e4a3 | |
parent | 8ddd995a2db096159312c2714c766c6c50485331 (diff) |
softirq: Adapt NOHZ softirq pending check to new RT scheme
We can't rely on ksoftirqd anymore and we need to check the tasks
which run a particular softirq and if such a task is pi blocked ignore
the other pending bits of that task as well.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
-rw-r--r-- | kernel/softirq.c | 68 |
1 files changed, 52 insertions, 16 deletions
diff --git a/kernel/softirq.c b/kernel/softirq.c index aaaadfbf3c79..c08332de2807 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -65,45 +65,75 @@ char *softirq_to_name[NR_SOFTIRQS] = { #ifdef CONFIG_NO_HZ # ifdef CONFIG_PREEMPT_RT_FULL + +struct softirq_runner { + struct task_struct *runner[NR_SOFTIRQS]; +}; + +static DEFINE_PER_CPU(struct softirq_runner, softirq_runners); + +static inline void softirq_set_runner(unsigned int sirq) +{ + struct softirq_runner *sr = &__get_cpu_var(softirq_runners); + + sr->runner[sirq] = current; +} + +static inline void softirq_clr_runner(unsigned int sirq) +{ + struct softirq_runner *sr = &__get_cpu_var(softirq_runners); + + sr->runner[sirq] = NULL; +} + /* - * On preempt-rt a softirq might be blocked on a lock. There might be - * no other runnable task on this CPU because the lock owner runs on - * some other CPU. So we have to go into idle with the pending bit - * set. Therefor we need to check this otherwise we warn about false - * positives which confuses users and defeats the whole purpose of - * this test. + * On preempt-rt a softirq running context might be blocked on a + * lock. There might be no other runnable task on this CPU because the + * lock owner runs on some other CPU. So we have to go into idle with + * the pending bit set. Therefor we need to check this otherwise we + * warn about false positives which confuses users and defeats the + * whole purpose of this test. * * This code is called with interrupts disabled. */ void softirq_check_pending_idle(void) { static int rate_limit; - u32 warnpending = 0, pending = local_softirq_pending(); + struct softirq_runner *sr = &__get_cpu_var(softirq_runners); + u32 warnpending, pending = local_softirq_pending(); if (rate_limit >= 10) return; - if (pending) { + warnpending = pending; + + while (pending) { struct task_struct *tsk; + int i = __ffs(pending); - tsk = __get_cpu_var(ksoftirqd); + pending &= ~(1 << i); + + tsk = sr->runner[i]; /* * The wakeup code in rtmutex.c wakes up the task * _before_ it sets pi_blocked_on to NULL under * tsk->pi_lock. So we need to check for both: state * and pi_blocked_on. */ - raw_spin_lock(&tsk->pi_lock); - - if (!tsk->pi_blocked_on && !(tsk->state == TASK_RUNNING)) - warnpending = 1; - - raw_spin_unlock(&tsk->pi_lock); + if (tsk) { + raw_spin_lock(&tsk->pi_lock); + if (tsk->pi_blocked_on || tsk->state == TASK_RUNNING) { + /* Clear all bits pending in that task */ + warnpending &= ~(tsk->softirqs_raised); + warnpending &= ~(1 << i); + } + raw_spin_unlock(&tsk->pi_lock); + } } if (warnpending) { printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n", - pending); + warnpending); rate_limit++; } } @@ -122,6 +152,10 @@ void softirq_check_pending_idle(void) } } # endif + +#else /* !NO_HZ */ +static inline void softirq_set_runner(unsigned int sirq) { } +static inline void softirq_clr_runner(unsigned int sirq) { } #endif /* @@ -482,6 +516,7 @@ static void do_current_softirqs(int need_rcu_bh_qs) */ lock_softirq(i); local_irq_disable(); + softirq_set_runner(i); /* * Check with the local_softirq_pending() bits, * whether we need to process this still or if someone @@ -492,6 +527,7 @@ static void do_current_softirqs(int need_rcu_bh_qs) set_softirq_pending(pending & ~mask); do_single_softirq(i, need_rcu_bh_qs); } + softirq_clr_runner(i); unlock_softirq(i); WARN_ON(current->softirq_nestcnt != 1); } |