proxmox-mirrors/mirror_ubuntu-kernels
1
0
Fork 0
mirror of https://git.proxmox.com/git/mirror_ubuntu-kernels.git synced 2025-11-29 11:13:44 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

srcu: Move work-scheduling fields from srcu_struct to srcu_usage

Browse Source 
This commit moves the ->reschedule_jiffies, ->reschedule_count, and
->work fields from the srcu_struct structure to the srcu_usage structure
to reduce the size of the former in order to improve cache locality.

However, this means that the container_of() calls cannot get a pointer
to the srcu_struct because they are no longer in the srcu_struct.
This issue is addressed by adding a ->srcu_ssp field in the srcu_usage
structure that references the corresponding srcu_struct structure.
And given the presence of the sup pointer to the srcu_usage structure,
replace some ssp->srcu_usage-> instances with sup->.

[ paulmck Apply feedback from kernel test robot. ]

Link: https://lore.kernel.org/oe-kbuild-all/202303191400.iO5BOqka-lkp@intel.com/
Suggested-by: Christoph Hellwig <hch@lst.de>
Tested-by: Sachin Sant <sachinp@linux.ibm.com>
Tested-by: "Zhang, Qiang1" <qiang1.zhang@intel.com>
Tested-by: Joel Fernandes (Google) <joel@joelfernandes.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
This commit is contained in:
Paul E. McKenney 2023-03-17 21:30:32 -07:00
parent d20162e0bf
commit fd1b3f8e09
2 changed files with 27 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
include/linux/srcutree.h
View File

@ -84,6 +84,10 @@ struct srcu_usage {
atomic_t srcu_barrier_cpu_cnt; /* # CPUs not yet posting a */ atomic_t srcu_barrier_cpu_cnt; /* # CPUs not yet posting a */
/* callback for the barrier */ /* callback for the barrier */
/* operation. */ /* operation. */
unsigned long reschedule_jiffies;
unsigned long reschedule_count;
struct delayed_work work;
struct srcu_struct *srcu_ssp;
}; };
/* /*
@ -92,9 +96,6 @@ struct srcu_usage {
struct srcu_struct { struct srcu_struct {
unsigned int srcu_idx; /* Current rdr array element. */ unsigned int srcu_idx; /* Current rdr array element. */
struct srcu_data __percpu *sda; /* Per-CPU srcu_data array. */ struct srcu_data __percpu *sda; /* Per-CPU srcu_data array. */
unsigned long reschedule_jiffies;
unsigned long reschedule_count;
struct delayed_work work;
struct lockdep_map dep_map; struct lockdep_map dep_map;
struct srcu_usage *srcu_sup; /* Update-side data. */ struct srcu_usage *srcu_sup; /* Update-side data. */
}; };
@ -119,10 +120,10 @@ struct srcu_struct {
{ \ { \
.lock = __SPIN_LOCK_UNLOCKED(name.lock), \ .lock = __SPIN_LOCK_UNLOCKED(name.lock), \
.srcu_gp_seq_needed = -1UL, \ .srcu_gp_seq_needed = -1UL, \
.work = __DELAYED_WORK_INITIALIZER(name.work, NULL, 0), \
} }
#define __SRCU_STRUCT_INIT_COMMON(name, usage_name) \ #define __SRCU_STRUCT_INIT_COMMON(name, usage_name) \
.work = __DELAYED_WORK_INITIALIZER(name.work, NULL, 0), \
.srcu_sup = &usage_name, \ .srcu_sup = &usage_name, \
__SRCU_DEP_MAP_INIT(name) __SRCU_DEP_MAP_INIT(name)

41
kernel/rcu/srcutree.c
View File

@ -251,7 +251,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
ssp->srcu_sup->srcu_barrier_seq = 0; ssp->srcu_sup->srcu_barrier_seq = 0;
mutex_init(&ssp->srcu_sup->srcu_barrier_mutex); mutex_init(&ssp->srcu_sup->srcu_barrier_mutex);
atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 0); atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 0);
INIT_DELAYED_WORK(&ssp->work, process_srcu); INIT_DELAYED_WORK(&ssp->srcu_sup->work, process_srcu);
ssp->srcu_sup->sda_is_static = is_static; ssp->srcu_sup->sda_is_static = is_static;
if (!is_static) if (!is_static)
ssp->sda = alloc_percpu(struct srcu_data); ssp->sda = alloc_percpu(struct srcu_data);
@ -275,6 +275,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_BIG); WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_BIG);
} }
} }
ssp->srcu_sup->srcu_ssp = ssp;
smp_store_release(&ssp->srcu_sup->srcu_gp_seq_needed, 0); /* Init done. */ smp_store_release(&ssp->srcu_sup->srcu_gp_seq_needed, 0); /* Init done. */
return 0; return 0;
} }
@ -647,7 +648,7 @@ void cleanup_srcu_struct(struct srcu_struct *ssp)
return; /* Just leak it! */ return; /* Just leak it! */
if (WARN_ON(srcu_readers_active(ssp))) if (WARN_ON(srcu_readers_active(ssp)))
return; /* Just leak it! */ return; /* Just leak it! */
flush_delayed_work(&ssp->work); flush_delayed_work(&ssp->srcu_sup->work);
for_each_possible_cpu(cpu) { for_each_possible_cpu(cpu) {
struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);
@ -1059,10 +1060,10 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp,
// can only be executed during early boot when there is only // can only be executed during early boot when there is only
// the one boot CPU running with interrupts still disabled. // the one boot CPU running with interrupts still disabled.
if (likely(srcu_init_done)) if (likely(srcu_init_done))
queue_delayed_work(rcu_gp_wq, &ssp->work, queue_delayed_work(rcu_gp_wq, &ssp->srcu_sup->work,
!!srcu_get_delay(ssp)); !!srcu_get_delay(ssp));
else if (list_empty(&ssp->work.work.entry)) else if (list_empty(&ssp->srcu_sup->work.work.entry))
list_add(&ssp->work.work.entry, &srcu_boot_list); list_add(&ssp->srcu_sup->work.work.entry, &srcu_boot_list);
} }
spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);
} }
@ -1723,7 +1724,7 @@ static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay)
spin_unlock_irq_rcu_node(ssp->srcu_sup); spin_unlock_irq_rcu_node(ssp->srcu_sup);
if (pushgp) if (pushgp)
queue_delayed_work(rcu_gp_wq, &ssp->work, delay); queue_delayed_work(rcu_gp_wq, &ssp->srcu_sup->work, delay);
} }
/* /*
@ -1734,22 +1735,24 @@ static void process_srcu(struct work_struct *work)
unsigned long curdelay; unsigned long curdelay;
unsigned long j; unsigned long j;
struct srcu_struct *ssp; struct srcu_struct *ssp;
struct srcu_usage *sup;
ssp = container_of(work, struct srcu_struct, work.work); sup = container_of(work, struct srcu_usage, work.work);
ssp = sup->srcu_ssp;
srcu_advance_state(ssp); srcu_advance_state(ssp);
curdelay = srcu_get_delay(ssp); curdelay = srcu_get_delay(ssp);
if (curdelay) { if (curdelay) {
WRITE_ONCE(ssp->reschedule_count, 0); WRITE_ONCE(sup->reschedule_count, 0);
} else { } else {
j = jiffies; j = jiffies;
if (READ_ONCE(ssp->reschedule_jiffies) == j) { if (READ_ONCE(sup->reschedule_jiffies) == j) {
WRITE_ONCE(ssp->reschedule_count, READ_ONCE(ssp->reschedule_count) + 1); WRITE_ONCE(sup->reschedule_count, READ_ONCE(sup->reschedule_count) + 1);
if (READ_ONCE(ssp->reschedule_count) > srcu_max_nodelay) if (READ_ONCE(sup->reschedule_count) > srcu_max_nodelay)
curdelay = 1; curdelay = 1;
} else { } else {
WRITE_ONCE(ssp->reschedule_count, 1); WRITE_ONCE(sup->reschedule_count, 1);
WRITE_ONCE(ssp->reschedule_jiffies, j); WRITE_ONCE(sup->reschedule_jiffies, j);
} }
} }
srcu_reschedule(ssp, curdelay); srcu_reschedule(ssp, curdelay);
@ -1848,7 +1851,7 @@ early_initcall(srcu_bootup_announce);
void __init srcu_init(void) void __init srcu_init(void)
{ {
struct srcu_struct *ssp; struct srcu_usage *sup;
/* Decide on srcu_struct-size strategy. */ /* Decide on srcu_struct-size strategy. */
if (SRCU_SIZING_IS(SRCU_SIZING_AUTO)) { if (SRCU_SIZING_IS(SRCU_SIZING_AUTO)) {
@ -1868,13 +1871,13 @@ void __init srcu_init(void)
*/ */
srcu_init_done = true; srcu_init_done = true;
while (!list_empty(&srcu_boot_list)) { while (!list_empty(&srcu_boot_list)) {
ssp = list_first_entry(&srcu_boot_list, struct srcu_struct, sup = list_first_entry(&srcu_boot_list, struct srcu_usage,
work.work.entry); work.work.entry);
list_del_init(&ssp->work.work.entry); list_del_init(&sup->work.work.entry);
if (SRCU_SIZING_IS(SRCU_SIZING_INIT) && if (SRCU_SIZING_IS(SRCU_SIZING_INIT) &&
ssp->srcu_sup->srcu_size_state == SRCU_SIZE_SMALL) sup->srcu_size_state == SRCU_SIZE_SMALL)
ssp->srcu_sup->srcu_size_state = SRCU_SIZE_ALLOC; sup->srcu_size_state = SRCU_SIZE_ALLOC;
queue_work(rcu_gp_wq, &ssp->work.work); queue_work(rcu_gp_wq, &sup->work.work);
} }
} }