1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
|
From 735b10844489babf52d3193193285a7311cf2c39 Mon Sep 17 00:00:00 2001
From: Juergen Gross <jgross@suse.com>
Date: Tue, 11 Oct 2022 15:14:22 +0200
Subject: [PATCH 058/126] xen/sched: introduce cpupool_update_node_affinity()
For updating the node affinities of all domains in a cpupool add a new
function cpupool_update_node_affinity().
In order to avoid multiple allocations of cpumasks carve out memory
allocation and freeing from domain_update_node_affinity() into new
helpers, which can be used by cpupool_update_node_affinity().
Modify domain_update_node_affinity() to take an additional parameter
for passing the allocated memory in and to allocate and free the memory
via the new helpers in case NULL was passed.
This will help later to pre-allocate the cpumasks in order to avoid
allocations in stop-machine context.
Signed-off-by: Juergen Gross <jgross@suse.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>
Acked-by: Andrew Cooper <andrew.cooper3@citrix.com>
Tested-by: Andrew Cooper <andrew.cooper3@citrix.com>
master commit: a83fa1e2b96ace65b45dde6954d67012633a082b
master date: 2022-09-05 11:42:30 +0100
---
xen/common/sched/core.c | 54 ++++++++++++++++++++++++++------------
xen/common/sched/cpupool.c | 39 +++++++++++++++------------
xen/common/sched/private.h | 7 +++++
xen/include/xen/sched.h | 9 ++++++-
4 files changed, 74 insertions(+), 35 deletions(-)
diff --git a/xen/common/sched/core.c b/xen/common/sched/core.c
index f07bd2681fcb..065a83eca912 100644
--- a/xen/common/sched/core.c
+++ b/xen/common/sched/core.c
@@ -1824,9 +1824,28 @@ int vcpu_affinity_domctl(struct domain *d, uint32_t cmd,
return ret;
}
-void domain_update_node_affinity(struct domain *d)
+bool alloc_affinity_masks(struct affinity_masks *affinity)
{
- cpumask_var_t dom_cpumask, dom_cpumask_soft;
+ if ( !alloc_cpumask_var(&affinity->hard) )
+ return false;
+ if ( !alloc_cpumask_var(&affinity->soft) )
+ {
+ free_cpumask_var(affinity->hard);
+ return false;
+ }
+
+ return true;
+}
+
+void free_affinity_masks(struct affinity_masks *affinity)
+{
+ free_cpumask_var(affinity->soft);
+ free_cpumask_var(affinity->hard);
+}
+
+void domain_update_node_aff(struct domain *d, struct affinity_masks *affinity)
+{
+ struct affinity_masks masks;
cpumask_t *dom_affinity;
const cpumask_t *online;
struct sched_unit *unit;
@@ -1836,14 +1855,16 @@ void domain_update_node_affinity(struct domain *d)
if ( !d->vcpu || !d->vcpu[0] )
return;
- if ( !zalloc_cpumask_var(&dom_cpumask) )
- return;
- if ( !zalloc_cpumask_var(&dom_cpumask_soft) )
+ if ( !affinity )
{
- free_cpumask_var(dom_cpumask);
- return;
+ affinity = &masks;
+ if ( !alloc_affinity_masks(affinity) )
+ return;
}
+ cpumask_clear(affinity->hard);
+ cpumask_clear(affinity->soft);
+
online = cpupool_domain_master_cpumask(d);
spin_lock(&d->node_affinity_lock);
@@ -1864,22 +1885,21 @@ void domain_update_node_affinity(struct domain *d)
*/
for_each_sched_unit ( d, unit )
{
- cpumask_or(dom_cpumask, dom_cpumask, unit->cpu_hard_affinity);
- cpumask_or(dom_cpumask_soft, dom_cpumask_soft,
- unit->cpu_soft_affinity);
+ cpumask_or(affinity->hard, affinity->hard, unit->cpu_hard_affinity);
+ cpumask_or(affinity->soft, affinity->soft, unit->cpu_soft_affinity);
}
/* Filter out non-online cpus */
- cpumask_and(dom_cpumask, dom_cpumask, online);
- ASSERT(!cpumask_empty(dom_cpumask));
+ cpumask_and(affinity->hard, affinity->hard, online);
+ ASSERT(!cpumask_empty(affinity->hard));
/* And compute the intersection between hard, online and soft */
- cpumask_and(dom_cpumask_soft, dom_cpumask_soft, dom_cpumask);
+ cpumask_and(affinity->soft, affinity->soft, affinity->hard);
/*
* If not empty, the intersection of hard, soft and online is the
* narrowest set we want. If empty, we fall back to hard&online.
*/
- dom_affinity = cpumask_empty(dom_cpumask_soft) ?
- dom_cpumask : dom_cpumask_soft;
+ dom_affinity = cpumask_empty(affinity->soft) ? affinity->hard
+ : affinity->soft;
nodes_clear(d->node_affinity);
for_each_cpu ( cpu, dom_affinity )
@@ -1888,8 +1908,8 @@ void domain_update_node_affinity(struct domain *d)
spin_unlock(&d->node_affinity_lock);
- free_cpumask_var(dom_cpumask_soft);
- free_cpumask_var(dom_cpumask);
+ if ( affinity == &masks )
+ free_affinity_masks(affinity);
}
typedef long ret_t;
diff --git a/xen/common/sched/cpupool.c b/xen/common/sched/cpupool.c
index 8c6e6eb9ccd5..45b6ff99561a 100644
--- a/xen/common/sched/cpupool.c
+++ b/xen/common/sched/cpupool.c
@@ -401,6 +401,25 @@ int cpupool_move_domain(struct domain *d, struct cpupool *c)
return ret;
}
+/* Update affinities of all domains in a cpupool. */
+static void cpupool_update_node_affinity(const struct cpupool *c)
+{
+ struct affinity_masks masks;
+ struct domain *d;
+
+ if ( !alloc_affinity_masks(&masks) )
+ return;
+
+ rcu_read_lock(&domlist_read_lock);
+
+ for_each_domain_in_cpupool(d, c)
+ domain_update_node_aff(d, &masks);
+
+ rcu_read_unlock(&domlist_read_lock);
+
+ free_affinity_masks(&masks);
+}
+
/*
* assign a specific cpu to a cpupool
* cpupool_lock must be held
@@ -408,7 +427,6 @@ int cpupool_move_domain(struct domain *d, struct cpupool *c)
static int cpupool_assign_cpu_locked(struct cpupool *c, unsigned int cpu)
{
int ret;
- struct domain *d;
const cpumask_t *cpus;
cpus = sched_get_opt_cpumask(c->gran, cpu);
@@ -433,12 +451,7 @@ static int cpupool_assign_cpu_locked(struct cpupool *c, unsigned int cpu)
rcu_read_unlock(&sched_res_rculock);
- rcu_read_lock(&domlist_read_lock);
- for_each_domain_in_cpupool(d, c)
- {
- domain_update_node_affinity(d);
- }
- rcu_read_unlock(&domlist_read_lock);
+ cpupool_update_node_affinity(c);
return 0;
}
@@ -447,18 +460,14 @@ static int cpupool_unassign_cpu_finish(struct cpupool *c)
{
int cpu = cpupool_moving_cpu;
const cpumask_t *cpus;
- struct domain *d;
int ret;
if ( c != cpupool_cpu_moving )
return -EADDRNOTAVAIL;
- /*
- * We need this for scanning the domain list, both in
- * cpu_disable_scheduler(), and at the bottom of this function.
- */
rcu_read_lock(&domlist_read_lock);
ret = cpu_disable_scheduler(cpu);
+ rcu_read_unlock(&domlist_read_lock);
rcu_read_lock(&sched_res_rculock);
cpus = get_sched_res(cpu)->cpus;
@@ -485,11 +494,7 @@ static int cpupool_unassign_cpu_finish(struct cpupool *c)
}
rcu_read_unlock(&sched_res_rculock);
- for_each_domain_in_cpupool(d, c)
- {
- domain_update_node_affinity(d);
- }
- rcu_read_unlock(&domlist_read_lock);
+ cpupool_update_node_affinity(c);
return ret;
}
diff --git a/xen/common/sched/private.h b/xen/common/sched/private.h
index 92d0d4961063..6e036f8c8077 100644
--- a/xen/common/sched/private.h
+++ b/xen/common/sched/private.h
@@ -593,6 +593,13 @@ affinity_balance_cpumask(const struct sched_unit *unit, int step,
cpumask_copy(mask, unit->cpu_hard_affinity);
}
+struct affinity_masks {
+ cpumask_var_t hard;
+ cpumask_var_t soft;
+};
+
+bool alloc_affinity_masks(struct affinity_masks *affinity);
+void free_affinity_masks(struct affinity_masks *affinity);
void sched_rm_cpu(unsigned int cpu);
const cpumask_t *sched_get_opt_cpumask(enum sched_gran opt, unsigned int cpu);
void schedule_dump(struct cpupool *c);
diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h
index 701963f84cb8..4e25627d9685 100644
--- a/xen/include/xen/sched.h
+++ b/xen/include/xen/sched.h
@@ -649,8 +649,15 @@ static inline void get_knownalive_domain(struct domain *d)
ASSERT(!(atomic_read(&d->refcnt) & DOMAIN_DESTROYED));
}
+struct affinity_masks;
+
int domain_set_node_affinity(struct domain *d, const nodemask_t *affinity);
-void domain_update_node_affinity(struct domain *d);
+void domain_update_node_aff(struct domain *d, struct affinity_masks *affinity);
+
+static inline void domain_update_node_affinity(struct domain *d)
+{
+ domain_update_node_aff(d, NULL);
+}
/*
* To be implemented by each architecture, sanity checking the configuration
--
2.37.4
|