Skip to content

Commit 867d447

Browse files
huaqiangwanghuaqiangwang
committed
objects: Introduce 'pcpuset' field for InstanceNUMACell
Introduce the 'pcpuset' to 'InstanceNUMACell' object to track the instance pinned CPUs. The 'InstanceNUMACell.cpuset' is switched to keep the instance unpinned CPUs only. As a result, the vCPUs of a dedicated instance is tracked in NUMA cell object's 'pcpuset', and vCPUs of a shared instance is put into the 'cpuset' field. This introduces some object data migration task for an existing instance that is in the 'dedicated' CPU allocation policy with the fact that all the CPUs are 1:1 pinned with host CPUs, and it requires to clear the content of 'InstanceNUMACell.cpuset' and move it to 'InstanceNUMACell.pcpuset' field. Part of blueprint use-pcpu-and-vcpu-in-one-instance Change-Id: I901fbd7df00e45196395ff4c69e7b8aa3359edf6 Signed-off-by: Stephen Finucane <[email protected]> Signed-off-by: Wang Huaqiang <[email protected]>
1 parent 55ff751 commit 867d447

File tree

22 files changed

+732
-407
lines changed

22 files changed

+732
-407
lines changed

‎nova/api/openstack/compute/server_topology.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ def _get_numa_topology(self, context, instance, show_host_info):
5353

5454
for cell_ in instance.numa_topology.cells:
5555
cell = {}
56-
cell['vcpu_set'] = cell_.cpuset
56+
cell['vcpu_set'] = cell_.total_cpus
5757
cell['siblings'] = cell_.siblings
5858
cell['memory_mb'] = cell_.memory
5959

‎nova/objects/instance_numa.py

Lines changed: 58 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,23 @@ class InstanceNUMACell(base.NovaEphemeralObject,
3333
# Version 1.2: Add cpu_pinning_raw and topology fields
3434
# Version 1.3: Add cpu_policy and cpu_thread_policy fields
3535
# Version 1.4: Add cpuset_reserved field
36-
VERSION = '1.4'
36+
# Version 1.5: Add pcpuset field
37+
VERSION = '1.5'
3738

3839
def obj_make_compatible(self, primitive, target_version):
3940
super(InstanceNUMACell, self).obj_make_compatible(primitive,
40-
target_version)
41+
target_version)
4142
target_version = versionutils.convert_version_to_tuple(target_version)
43+
# NOTE(huaqiang): Since version 1.5, 'cpuset' is modified to track the
44+
# unpinned CPUs only, with pinned CPUs tracked via 'pcpuset' instead.
45+
# For a backward compatibility, move the 'dedicated' instance CPU list
46+
# from 'pcpuset' to 'cpuset'.
47+
if target_version < (1, 5):
48+
if (primitive['cpu_policy'] ==
49+
obj_fields.CPUAllocationPolicy.DEDICATED):
50+
primitive['cpuset'] = primitive['pcpuset']
51+
primitive.pop('pcpuset', None)
52+
4253
if target_version < (1, 4):
4354
primitive.pop('cpuset_reserved', None)
4455

@@ -49,6 +60,10 @@ def obj_make_compatible(self, primitive, target_version):
4960
fields = {
5061
'id': obj_fields.IntegerField(),
5162
'cpuset': obj_fields.SetOfIntegersField(),
63+
'pcpuset': obj_fields.SetOfIntegersField(),
64+
# These physical CPUs are reserved for use by the hypervisor
65+
'cpuset_reserved': obj_fields.SetOfIntegersField(nullable=True,
66+
default=None),
5267
'memory': obj_fields.IntegerField(),
5368
'pagesize': obj_fields.IntegerField(nullable=True,
5469
default=None),
@@ -60,19 +75,20 @@ def obj_make_compatible(self, primitive, target_version):
6075
default=None),
6176
'cpu_thread_policy': obj_fields.CPUThreadAllocationPolicyField(
6277
nullable=True, default=None),
63-
# These physical CPUs are reserved for use by the hypervisor
64-
'cpuset_reserved': obj_fields.SetOfIntegersField(nullable=True,
65-
default=None),
6678
}
6779

6880
cpu_pinning = obj_fields.DictProxyField('cpu_pinning_raw')
6981

7082
def __len__(self):
71-
return len(self.cpuset)
83+
return len(self.total_cpus)
84+
85+
@property
86+
def total_cpus(self):
87+
return self.cpuset | self.pcpuset
7288

7389
@property
7490
def siblings(self):
75-
cpu_list = sorted(list(self.cpuset))
91+
cpu_list = sorted(list(self.total_cpus))
7692

7793
threads = 0
7894
if ('cpu_topology' in self) and self.cpu_topology:
@@ -83,7 +99,7 @@ def siblings(self):
8399
return list(map(set, zip(*[iter(cpu_list)] * threads)))
84100

85101
def pin(self, vcpu, pcpu):
86-
if vcpu not in self.cpuset:
102+
if vcpu not in self.pcpuset:
87103
return
88104
pinning_dict = self.cpu_pinning or {}
89105
pinning_dict[vcpu] = pcpu
@@ -115,7 +131,7 @@ class InstanceNUMATopology(base.NovaObject,
115131

116132
def obj_make_compatible(self, primitive, target_version):
117133
super(InstanceNUMATopology, self).obj_make_compatible(primitive,
118-
target_version)
134+
target_version)
119135
target_version = versionutils.convert_version_to_tuple(target_version)
120136
if target_version < (1, 3):
121137
primitive.pop('emulator_threads_policy', None)
@@ -136,11 +152,43 @@ def obj_from_db_obj(cls, context, instance_uuid, db_obj):
136152

137153
if 'nova_object.name' in primitive:
138154
obj = cls.obj_from_primitive(primitive)
155+
cls._migrate_legacy_dedicated_instance_cpuset(
156+
context, instance_uuid, obj)
139157
else:
140158
obj = cls._migrate_legacy_object(context, instance_uuid, primitive)
141159

142160
return obj
143161

162+
# TODO(huaqiang): Remove after Wallaby once we are sure these objects have
163+
# been loaded at least once.
164+
@classmethod
165+
def _migrate_legacy_dedicated_instance_cpuset(cls, context, instance_uuid,
166+
obj):
167+
# NOTE(huaqiang): We may meet some topology object with the old version
168+
# 'InstanceNUMACell' cells, in that case, the 'dedicated' CPU is kept
169+
# in 'InstanceNUMACell.cpuset' field, but it should be kept in
170+
# 'InstanceNUMACell.pcpuset' field since Victoria. Making an upgrade
171+
# and persisting to database.
172+
update_db = False
173+
for cell in obj.cells:
174+
if len(cell.cpuset) == 0:
175+
continue
176+
177+
if cell.cpu_policy != obj_fields.CPUAllocationPolicy.DEDICATED:
178+
continue
179+
180+
cell.pcpuset = cell.cpuset
181+
cell.cpuset = set()
182+
update_db = True
183+
184+
if update_db:
185+
db_obj = jsonutils.dumps(obj.obj_to_primitive())
186+
values = {
187+
'numa_topology': db_obj,
188+
}
189+
db.instance_extra_update_by_uuid(context, instance_uuid,
190+
values)
191+
144192
# TODO(stephenfin): Remove in X or later, once this has bedded in
145193
@classmethod
146194
def _migrate_legacy_object(cls, context, instance_uuid, primitive):
@@ -161,6 +209,7 @@ def _migrate_legacy_object(cls, context, instance_uuid, primitive):
161209
InstanceNUMACell(
162210
id=cell.get('id'),
163211
cpuset=hardware.parse_cpu_spec(cell.get('cpus', '')),
212+
pcpuset=set(),
164213
memory=cell.get('mem', {}).get('total', 0),
165214
pagesize=cell.get('pagesize'),
166215
) for cell in primitive.get('cells', [])

‎nova/tests/functional/api_sample_tests/test_server_topology.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,14 @@ def fake_get_numa():
2222
cell_0 = numa.InstanceNUMACell(node=0, memory=1024, pagesize=4, id=0,
2323
cpu_topology=cpu_topology,
2424
cpu_pinning={0: 0, 1: 5},
25-
cpuset=set([0, 1]))
25+
cpuset=set(),
26+
pcpuset=set([0, 1]))
2627

2728
cell_1 = numa.InstanceNUMACell(node=1, memory=2048, pagesize=4, id=1,
2829
cpu_topology=cpu_topology,
2930
cpu_pinning={2: 1, 3: 8},
30-
cpuset=set([2, 3]))
31+
cpuset=set(),
32+
pcpuset=set([2, 3]))
3133

3234
return numa.InstanceNUMATopology(cells=[cell_0, cell_1])
3335

‎nova/tests/unit/api/openstack/compute/test_server_topology.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,8 @@ def setUp(self):
4040

4141
def _fake_numa(self, cpu_pinning=None):
4242
ce0 = numa.InstanceNUMACell(node=0, memory=1024, pagesize=4, id=0,
43-
cpu_topology=None,
44-
cpu_pinning=cpu_pinning,
45-
cpuset=set([0, 1]))
43+
cpu_topology=None, cpu_pinning=cpu_pinning,
44+
cpuset=set([0, 1]), pcpuset=set())
4645

4746
return numa.InstanceNUMATopology(cells=[ce0])
4847

‎nova/tests/unit/compute/test_claims.py

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -186,13 +186,14 @@ def test_pci_pass_no_requests(self, mock_pci_supports_requests):
186186
def test_numa_topology_no_limit(self):
187187
huge_instance = objects.InstanceNUMATopology(
188188
cells=[objects.InstanceNUMACell(
189-
id=1, cpuset=set([1, 2]), memory=512)])
189+
id=1, cpuset=set([1, 2]), pcpuset=set(), memory=512)])
190190
self._claim(numa_topology=huge_instance)
191191

192192
def test_numa_topology_fails(self):
193193
huge_instance = objects.InstanceNUMATopology(
194194
cells=[objects.InstanceNUMACell(
195-
id=1, cpuset=set([1, 2, 3, 4, 5]), memory=2048)])
195+
id=1, cpuset=set([1, 2, 3, 4, 5]), pcpuset=set(),
196+
memory=2048)])
196197
limit_topo = objects.NUMATopologyLimits(
197198
cpu_allocation_ratio=1, ram_allocation_ratio=1)
198199
self.assertRaises(exception.ComputeResourcesUnavailable,
@@ -203,7 +204,7 @@ def test_numa_topology_fails(self):
203204
def test_numa_topology_passes(self):
204205
huge_instance = objects.InstanceNUMATopology(
205206
cells=[objects.InstanceNUMACell(
206-
id=1, cpuset=set([1, 2]), memory=512)])
207+
id=1, cpuset=set([1, 2]), pcpuset=set(), memory=512)])
207208
limit_topo = objects.NUMATopologyLimits(
208209
cpu_allocation_ratio=1, ram_allocation_ratio=1)
209210
self._claim(limits={'numa_topology': limit_topo},
@@ -230,7 +231,7 @@ def test_numa_topology_with_pci(self, mock_get_by_instance):
230231

231232
huge_instance = objects.InstanceNUMATopology(
232233
cells=[objects.InstanceNUMACell(
233-
id=1, cpuset=set([1, 2]), memory=512)])
234+
id=1, cpuset=set([1, 2]), pcpuset=set(), memory=512)])
234235

235236
self._claim(requests=requests, numa_topology=huge_instance)
236237

@@ -265,7 +266,7 @@ def test_numa_topology_with_pci_fail(self, mock_get_by_instance):
265266

266267
huge_instance = objects.InstanceNUMATopology(
267268
cells=[objects.InstanceNUMACell(
268-
id=1, cpuset=set([1, 2]), memory=512)])
269+
id=1, cpuset=set([1, 2]), pcpuset=set(), memory=512)])
269270

270271
self.assertRaises(exception.ComputeResourcesUnavailable,
271272
self._claim,
@@ -294,7 +295,7 @@ def test_numa_topology_with_pci_no_numa_info(self, mock_get_by_instance):
294295

295296
huge_instance = objects.InstanceNUMATopology(
296297
cells=[objects.InstanceNUMACell(
297-
id=1, cpuset=set([1, 2]), memory=512)])
298+
id=1, cpuset=set([1, 2]), pcpuset=set(), memory=512)])
298299

299300
self._claim(requests=requests, numa_topology=huge_instance)
300301

@@ -381,11 +382,12 @@ def test_live_migration_page_size(self):
381382
instance_type = self._fake_instance_type()
382383
instance = self._fake_instance()
383384
instance.numa_topology = objects.InstanceNUMATopology(
384-
cells=[objects.InstanceNUMACell(id=1, cpuset=set([1, 2]),
385-
memory=512, pagesize=2)])
385+
cells=[objects.InstanceNUMACell(
386+
id=1, cpuset=set([1, 2]),
387+
pcpuset=set(), memory=512, pagesize=2)])
386388
claimed_numa_topology = objects.InstanceNUMATopology(
387-
cells=[objects.InstanceNUMACell(id=1, cpuset=set([1, 2]),
388-
memory=512, pagesize=1)])
389+
cells=[objects.InstanceNUMACell(
390+
id=1, cpuset=set([1, 2]), pcpuset=set(), memory=512, pagesize=1)])
389391
with mock.patch('nova.virt.hardware.numa_fit_instance_to_host',
390392
return_value=claimed_numa_topology):
391393
self.assertRaisesRegex(
@@ -402,8 +404,9 @@ def test_claim_fails_page_size_not_called(self):
402404
# This topology cannot fit in self.compute_node
403405
# (see _fake_compute_node())
404406
numa_topology = objects.InstanceNUMATopology(
405-
cells=[objects.InstanceNUMACell(id=1, cpuset=set([1, 2, 3]),
406-
memory=1024)])
407+
cells=[objects.InstanceNUMACell(
408+
id=1, cpuset=set([1, 2, 3]), pcpuset=set(),
409+
memory=1024)])
407410
with test.nested(
408411
mock.patch('nova.virt.hardware.numa_get_constraints',
409412
return_value=numa_topology),

‎nova/tests/unit/compute/test_compute.py

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5682,7 +5682,8 @@ def test_confirm_resize_with_numa_topology_and_cpu_pinning(
56825682
old_inst_topology = objects.InstanceNUMATopology(
56835683
instance_uuid=instance.uuid, cells=[
56845684
objects.InstanceNUMACell(
5685-
id=0, cpuset=set([1, 2]), memory=512, pagesize=2048,
5685+
id=0, cpuset=set(), pcpuset=set([1, 2]), memory=512,
5686+
pagesize=2048,
56865687
cpu_policy=obj_fields.CPUAllocationPolicy.DEDICATED,
56875688
cpu_pinning={'0': 1, '1': 2})
56885689
])
@@ -5691,7 +5692,8 @@ def test_confirm_resize_with_numa_topology_and_cpu_pinning(
56915692
new_inst_topology = objects.InstanceNUMATopology(
56925693
instance_uuid=instance.uuid, cells=[
56935694
objects.InstanceNUMACell(
5694-
id=1, cpuset=set([3, 4]), memory=512, pagesize=2048,
5695+
id=1, cpuset=set(), pcpuset=set([3, 4]), memory=512,
5696+
pagesize=2048,
56955697
cpu_policy=obj_fields.CPUAllocationPolicy.DEDICATED,
56965698
cpu_pinning={'0': 3, '1': 4})
56975699
])
@@ -8694,9 +8696,10 @@ def test_create_with_deleted_image(self):
86948696
def test_create_with_numa_topology(self, numa_constraints_mock):
86958697
numa_topology = objects.InstanceNUMATopology(
86968698
cells=[objects.InstanceNUMACell(
8697-
id=0, cpuset=set([1, 2]), memory=512),
8699+
id=0, cpuset=set([1, 2]), pcpuset=set(),
8700+
memory=512),
86988701
objects.InstanceNUMACell(
8699-
id=1, cpuset=set([3, 4]), memory=512)])
8702+
id=1, cpuset=set([3, 4]), pcpuset=set(), memory=512)])
87008703
numa_topology.obj_reset_changes()
87018704
numa_constraints_mock.return_value = numa_topology
87028705

‎nova/tests/unit/compute/test_compute_api.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1692,7 +1692,8 @@ def _test_revert_resize(
16921692
fake_reqspec.flavor = fake_inst.flavor
16931693
fake_numa_topology = objects.InstanceNUMATopology(cells=[
16941694
objects.InstanceNUMACell(
1695-
id=0, cpuset=set([0]), memory=512, pagesize=None,
1695+
id=0, cpuset=set([0]), pcpuset=set(), memory=512,
1696+
pagesize=None,
16961697
cpu_pinning_raw=None, cpuset_reserved=None, cpu_policy=None,
16971698
cpu_thread_policy=None)])
16981699

@@ -1852,7 +1853,8 @@ def _test_resize(self, mock_get_all_by_host,
18521853
fake_inst = self._create_instance_obj(params=params)
18531854
fake_numa_topology = objects.InstanceNUMATopology(cells=[
18541855
objects.InstanceNUMACell(
1855-
id=0, cpuset=set([0]), memory=512, pagesize=None,
1856+
id=0, cpuset=set([0]), pcpuset=set(), memory=512,
1857+
pagesize=None,
18561858
cpu_pinning_raw=None, cpuset_reserved=None, cpu_policy=None,
18571859
cpu_thread_policy=None)])
18581860

‎nova/tests/unit/compute/test_compute_mgr.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1206,14 +1206,18 @@ def _test__validate_pinning_configuration(self, supports_pcpus=True):
12061206

12071207
numa_wo_pinning = test_instance_numa.get_fake_obj_numa_topology(
12081208
self.context)
1209+
numa_wo_pinning.cells[0].pcpuset = set()
1210+
numa_wo_pinning.cells[1].pcpuset = set()
12091211
instance_2.numa_topology = numa_wo_pinning
12101212

12111213
numa_w_pinning = test_instance_numa.get_fake_obj_numa_topology(
12121214
self.context)
12131215
numa_w_pinning.cells[0].pin_vcpus((1, 10), (2, 11))
1216+
numa_w_pinning.cells[0].cpuset = set()
12141217
numa_w_pinning.cells[0].cpu_policy = (
12151218
fields.CPUAllocationPolicy.DEDICATED)
12161219
numa_w_pinning.cells[1].pin_vcpus((3, 0), (4, 1))
1220+
numa_w_pinning.cells[1].cpuset = set()
12171221
numa_w_pinning.cells[1].cpu_policy = (
12181222
fields.CPUAllocationPolicy.DEDICATED)
12191223
instance_3.numa_topology = numa_w_pinning

‎nova/tests/unit/compute/test_resource_tracker.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -153,9 +153,9 @@
153153
_INSTANCE_NUMA_TOPOLOGIES = {
154154
'2mb': objects.InstanceNUMATopology(cells=[
155155
objects.InstanceNUMACell(
156-
id=0, cpuset=set([1]), memory=_2MB, pagesize=0),
156+
id=0, cpuset=set([1]), pcpuset=set(), memory=_2MB, pagesize=0),
157157
objects.InstanceNUMACell(
158-
id=1, cpuset=set([3]), memory=_2MB, pagesize=0)]),
158+
id=1, cpuset=set([3]), pcpuset=set(), memory=_2MB, pagesize=0)]),
159159
}
160160

161161
_NUMA_LIMIT_TOPOLOGIES = {

‎nova/tests/unit/conductor/tasks/test_live_migrate.py

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -214,9 +214,10 @@ def test_check_instance_has_no_numa_passes_no_numa(self, mock_get):
214214
@mock.patch.object(objects.ComputeNode, 'get_by_host_and_nodename')
215215
def test_check_instance_has_no_numa_passes_non_kvm(self, mock_get):
216216
self.flags(enable_numa_live_migration=False, group='workarounds')
217-
self.task.instance.numa_topology = objects.InstanceNUMATopology(
218-
cells=[objects.InstanceNUMACell(id=0, cpuset=set([0]),
219-
memory=1024)])
217+
self.task.instance.numa_topology = objects.InstanceNUMATopology(cells=[
218+
objects.InstanceNUMACell(
219+
id=0, cpuset=set([0]), pcpuset=set(), memory=1024),
220+
])
220221
mock_get.return_value = objects.ComputeNode(
221222
uuid=uuids.cn1, hypervisor_type='xen')
222223
self.task._check_instance_has_no_numa()
@@ -227,9 +228,10 @@ def test_check_instance_has_no_numa_passes_non_kvm(self, mock_get):
227228
def test_check_instance_has_no_numa_passes_workaround(
228229
self, mock_get_min_ver, mock_get):
229230
self.flags(enable_numa_live_migration=True, group='workarounds')
230-
self.task.instance.numa_topology = objects.InstanceNUMATopology(
231-
cells=[objects.InstanceNUMACell(id=0, cpuset=set([0]),
232-
memory=1024)])
231+
self.task.instance.numa_topology = objects.InstanceNUMATopology(cells=[
232+
objects.InstanceNUMACell(
233+
id=0, cpuset=set([0]), pcpuset=set(), memory=1024),
234+
])
233235
mock_get.return_value = objects.ComputeNode(
234236
uuid=uuids.cn1, hypervisor_type='qemu')
235237
self.task._check_instance_has_no_numa()
@@ -243,9 +245,10 @@ def test_check_instance_has_no_numa_fails(self, mock_get_min_ver,
243245
self.flags(enable_numa_live_migration=False, group='workarounds')
244246
mock_get.return_value = objects.ComputeNode(
245247
uuid=uuids.cn1, hypervisor_type='qemu')
246-
self.task.instance.numa_topology = objects.InstanceNUMATopology(
247-
cells=[objects.InstanceNUMACell(id=0, cpuset=set([0]),
248-
memory=1024)])
248+
self.task.instance.numa_topology = objects.InstanceNUMATopology(cells=[
249+
objects.InstanceNUMACell(
250+
id=0, cpuset=set([0]), pcpuset=set(), memory=1024),
251+
])
249252
self.assertRaises(exception.MigrationPreCheckError,
250253
self.task._check_instance_has_no_numa)
251254
mock_get_min_ver.assert_called_once_with(self.context, 'nova-compute')
@@ -258,9 +261,10 @@ def test_check_instance_has_no_numa_new_svc_passes(self, mock_get_min_ver,
258261
self.flags(enable_numa_live_migration=False, group='workarounds')
259262
mock_get.return_value = objects.ComputeNode(
260263
uuid=uuids.cn1, hypervisor_type='qemu')
261-
self.task.instance.numa_topology = objects.InstanceNUMATopology(
262-
cells=[objects.InstanceNUMACell(id=0, cpuset=set([0]),
263-
memory=1024)])
264+
self.task.instance.numa_topology = objects.InstanceNUMATopology(cells=[
265+
objects.InstanceNUMACell(
266+
id=0, cpuset=set([0]), pcpuset=set(), memory=1024),
267+
])
264268
self.task._check_instance_has_no_numa()
265269
mock_get_min_ver.assert_called_once_with(self.context, 'nova-compute')
266270

0 commit comments

Comments
 (0)