diff options
| author |  Paul E. McKenney <paulmck@kernel.org>
| 2021-11-23 16:16:50 -0800 |
|---|---|---|
| committer | Paul E. McKenney <paulmck@kernel.org>
| 2021-12-09 10:52:11 -0800 |
| commit | 3063b33a347c088e87516764d487e46fea3dfc94 (patch) | |
| tree | c32d61232109eb2faf19d4afbbd5d1ea17c17e89 /kernel/rcu | |
| parent | rcu-tasks: Count trylocks to estimate call_rcu_tasks() contention (diff) | |
| download | linux-3063b33a347c088e87516764d487e46fea3dfc94.tar.gz linux-3063b33a347c088e87516764d487e46fea3dfc94.tar.bz2 linux-3063b33a347c088e87516764d487e46fea3dfc94.zip | |
rcu-tasks: Avoid raw-spinlocked wakeups from call_rcu_tasks_generic()
If the caller of of call_rcu_tasks(), call_rcu_tasks_rude(),
or call_rcu_tasks_trace() holds a raw spinlock, and then if
call_rcu_tasks_generic() determines that the grace-period kthread must
be awakened, then the wakeup might acquire a normal spinlock while a
raw spinlock is held. This results in lockdep splats when the
kernel is built with CONFIG_PROVE_RAW_LOCK_NESTING=y.
This commit therefore defers the wakeup using irq_work_queue().
It would be nice to directly invoke wakeup when a raw spinlock is not
held, but there is currently no way to check for this in all kernels.
Reported-by: Martin Lau <kafai@fb.com>
Cc: Neeraj Upadhyay <neeraj.iitr10@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Diffstat (limited to 'kernel/rcu')
| -rw-r--r-- | kernel/rcu/tasks.h | 17 |
1 files changed, 16 insertions, 1 deletions
diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 231a949cdb81..6e8e4cf6fff3 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -27,6 +27,7 @@ typedef void (*postgp_func_t)(struct rcu_tasks *rtp); * @rtp_jiffies: Jiffies counter value for statistics. * @rtp_n_lock_retries: Rough lock-contention statistic. * @rtp_work: Work queue for invoking callbacks. + * @rtp_irq_work: IRQ work queue for deferred wakeups. * @barrier_q_head: RCU callback for barrier operation. * @cpu: CPU number corresponding to this entry. * @rtpp: Pointer to the rcu_tasks structure. @@ -37,6 +38,7 @@ struct rcu_tasks_percpu { unsigned long rtp_jiffies; unsigned long rtp_n_lock_retries; struct work_struct rtp_work; + struct irq_work rtp_irq_work; struct rcu_head barrier_q_head; int cpu; struct rcu_tasks *rtpp; @@ -102,9 +104,12 @@ struct rcu_tasks { char *kname; }; +static void call_rcu_tasks_iw_wakeup(struct irq_work *iwp); + #define DEFINE_RCU_TASKS(rt_name, gp, call, n) \ static DEFINE_PER_CPU(struct rcu_tasks_percpu, rt_name ## __percpu) = { \ .lock = __RAW_SPIN_LOCK_UNLOCKED(rt_name ## __percpu.cbs_pcpu_lock), \ + .rtp_irq_work = IRQ_WORK_INIT(call_rcu_tasks_iw_wakeup), \ }; \ static struct rcu_tasks rt_name = \ { \ @@ -230,6 +235,16 @@ static void cblist_init_generic(struct rcu_tasks *rtp) pr_info("%s: Setting shift to %d and lim to %d.\n", __func__, data_race(rtp->percpu_enqueue_shift), data_race(rtp->percpu_enqueue_lim)); } +// IRQ-work handler that does deferred wakeup for call_rcu_tasks_generic(). +static void call_rcu_tasks_iw_wakeup(struct irq_work *iwp) +{ + struct rcu_tasks *rtp; + struct rcu_tasks_percpu *rtpcp = container_of(iwp, struct rcu_tasks_percpu, rtp_irq_work); + + rtp = rtpcp->rtpp; + wake_up(&rtp->cbs_wq); +} + // Enqueue a callback for the specified flavor of Tasks RCU. static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, struct rcu_tasks *rtp) @@ -263,7 +278,7 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); /* We can't create the thread unless interrupts are enabled. */ if (needwake && READ_ONCE(rtp->kthread_ptr)) - wake_up(&rtp->cbs_wq); + irq_work_queue(&rtpcp->rtp_irq_work); } // Wait for a grace period for the specified flavor of Tasks RCU. |