Skip to content

Commit 297f3ba

Browse files
author
Adam Spiers
committed
Add infrastructure for invoking libvirt's getDomainCapabilities API
Two use cases have emerged semi-recently which both require the libvirt driver to be able to invoke libvirt's virConnectGetDomainCapabilities() API: https://libvirt.org/html/libvirt-libvirt-domain.html#virConnectGetDomainCapabilities and parse the results: - Automatic detection of AMD compute hosts which are capable of providing SEV (Secure Encrypted Virtualization) - Gracefully handling different QEMU machine types for x86 hosts So lay the foundation for these use cases by adding a new get_domain_capabilities() method to nova.virt.libvirt.host.Host, along with new subclasses of LibvirtConfigObject for parsing the XML returned from libvirt, and corresponding tests. Change-Id: I4aeac9b2397bb2f5e130d1e58829a5e549fcb191 blueprint: gracefully-handle-qemu-machine-types blueprint: amd-sev-libvirt-support
1 parent ed5b7c7 commit 297f3ba

File tree

5 files changed

+341
-0
lines changed

5 files changed

+341
-0
lines changed

‎nova/tests/unit/virt/libvirt/fakelibvirt.py

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1357,6 +1357,154 @@ def getCPUMap(self):
13571357
else False for cpu_num in range(total_cpus)]
13581358
return (total_cpus, cpu_map, active_cpus)
13591359

1360+
def getDomainCapabilities(self, emulatorbin, arch, machine_type,
1361+
virt_type, flags):
1362+
"""Return spoofed domain capabilities."""
1363+
1364+
return '''
1365+
<domainCapabilities>
1366+
<path>/usr/bin/qemu-kvm</path>
1367+
<domain>kvm</domain>
1368+
<machine>pc-i440fx-2.11</machine>
1369+
<arch>x86_64</arch>
1370+
<vcpu max='255'/>
1371+
<os supported='yes'>
1372+
<loader supported='yes'>
1373+
<value>/usr/share/qemu/ovmf-x86_64-ms-4m-code.bin</value>
1374+
<value>/usr/share/qemu/ovmf-x86_64-ms-code.bin</value>
1375+
<enum name='type'>
1376+
<value>rom</value>
1377+
<value>pflash</value>
1378+
</enum>
1379+
<enum name='readonly'>
1380+
<value>yes</value>
1381+
<value>no</value>
1382+
</enum>
1383+
</loader>
1384+
</os>
1385+
<cpu>
1386+
<mode name='host-passthrough' supported='yes'/>
1387+
<mode name='host-model' supported='yes'>
1388+
<model fallback='forbid'>EPYC-IBPB</model>
1389+
<vendor>AMD</vendor>
1390+
<feature policy='require' name='x2apic'/>
1391+
<feature policy='require' name='tsc-deadline'/>
1392+
<feature policy='require' name='hypervisor'/>
1393+
<feature policy='require' name='tsc_adjust'/>
1394+
<feature policy='require' name='cmp_legacy'/>
1395+
<feature policy='require' name='invtsc'/>
1396+
<feature policy='require' name='virt-ssbd'/>
1397+
<feature policy='disable' name='monitor'/>
1398+
</mode>
1399+
<mode name='custom' supported='yes'>
1400+
<model usable='yes'>qemu64</model>
1401+
<model usable='yes'>qemu32</model>
1402+
<model usable='no'>phenom</model>
1403+
<model usable='yes'>pentium3</model>
1404+
<model usable='yes'>pentium2</model>
1405+
<model usable='yes'>pentium</model>
1406+
<model usable='no'>n270</model>
1407+
<model usable='yes'>kvm64</model>
1408+
<model usable='yes'>kvm32</model>
1409+
<model usable='no'>coreduo</model>
1410+
<model usable='no'>core2duo</model>
1411+
<model usable='no'>athlon</model>
1412+
<model usable='yes'>Westmere</model>
1413+
<model usable='no'>Westmere-IBRS</model>
1414+
<model usable='no'>Skylake-Server</model>
1415+
<model usable='no'>Skylake-Server-IBRS</model>
1416+
<model usable='no'>Skylake-Client</model>
1417+
<model usable='no'>Skylake-Client-IBRS</model>
1418+
<model usable='yes'>SandyBridge</model>
1419+
<model usable='no'>SandyBridge-IBRS</model>
1420+
<model usable='yes'>Penryn</model>
1421+
<model usable='no'>Opteron_G5</model>
1422+
<model usable='no'>Opteron_G4</model>
1423+
<model usable='yes'>Opteron_G3</model>
1424+
<model usable='yes'>Opteron_G2</model>
1425+
<model usable='yes'>Opteron_G1</model>
1426+
<model usable='yes'>Nehalem</model>
1427+
<model usable='no'>Nehalem-IBRS</model>
1428+
<model usable='no'>IvyBridge</model>
1429+
<model usable='no'>IvyBridge-IBRS</model>
1430+
<model usable='no'>Haswell</model>
1431+
<model usable='no'>Haswell-noTSX</model>
1432+
<model usable='no'>Haswell-noTSX-IBRS</model>
1433+
<model usable='no'>Haswell-IBRS</model>
1434+
<model usable='yes'>EPYC</model>
1435+
<model usable='yes'>EPYC-IBPB</model>
1436+
<model usable='yes'>Conroe</model>
1437+
<model usable='no'>Broadwell</model>
1438+
<model usable='no'>Broadwell-noTSX</model>
1439+
<model usable='no'>Broadwell-noTSX-IBRS</model>
1440+
<model usable='no'>Broadwell-IBRS</model>
1441+
<model usable='yes'>486</model>
1442+
</mode>
1443+
</cpu>
1444+
<devices>
1445+
<disk supported='yes'>
1446+
<enum name='diskDevice'>
1447+
<value>disk</value>
1448+
<value>cdrom</value>
1449+
<value>floppy</value>
1450+
<value>lun</value>
1451+
</enum>
1452+
<enum name='bus'>
1453+
<value>ide</value>
1454+
<value>fdc</value>
1455+
<value>scsi</value>
1456+
<value>virtio</value>
1457+
<value>usb</value>
1458+
<value>sata</value>
1459+
</enum>
1460+
</disk>
1461+
<graphics supported='yes'>
1462+
<enum name='type'>
1463+
<value>sdl</value>
1464+
<value>vnc</value>
1465+
<value>spice</value>
1466+
</enum>
1467+
</graphics>
1468+
<video supported='yes'>
1469+
<enum name='modelType'>
1470+
<value>vga</value>
1471+
<value>cirrus</value>
1472+
<value>vmvga</value>
1473+
<value>qxl</value>
1474+
<value>virtio</value>
1475+
</enum>
1476+
</video>
1477+
<hostdev supported='yes'>
1478+
<enum name='mode'>
1479+
<value>subsystem</value>
1480+
</enum>
1481+
<enum name='startupPolicy'>
1482+
<value>default</value>
1483+
<value>mandatory</value>
1484+
<value>requisite</value>
1485+
<value>optional</value>
1486+
</enum>
1487+
<enum name='subsysType'>
1488+
<value>usb</value>
1489+
<value>pci</value>
1490+
<value>scsi</value>
1491+
</enum>
1492+
<enum name='capsType'/>
1493+
<enum name='pciBackend'>
1494+
<value>default</value>
1495+
<value>vfio</value>
1496+
</enum>
1497+
</hostdev>
1498+
</devices>
1499+
%(features)s
1500+
</domainCapabilities>''' % {'features': self._domain_capability_features}
1501+
1502+
# Features are kept separately so that the tests can patch this
1503+
# class variable with alternate values.
1504+
_domain_capability_features = ''' <features>
1505+
<gic supported='no'/>
1506+
</features>'''
1507+
13601508
def getCapabilities(self):
13611509
"""Return spoofed capabilities."""
13621510
numa_topology = self.host_info.numa_topology

‎nova/tests/unit/virt/libvirt/test_fakelibvirt.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,11 @@ def test_getCapabilities(self):
277277
conn = self.get_openAuth_curry_func()('qemu:///system')
278278
etree.fromstring(conn.getCapabilities())
279279

280+
def test_getDomainCapabilities(self):
281+
conn = self.get_openAuth_curry_func()('qemu:///system')
282+
etree.fromstring(conn.getDomainCapabilities(
283+
'/usr/bin/qemu-kvm', 'x86_64', 'q35', 'kvm', 0))
284+
280285
def test_nwfilter_define_undefine(self):
281286
conn = self.get_openAuth_curry_func()('qemu:///system')
282287
# Will raise an exception if it's not valid XML

‎nova/tests/unit/virt/libvirt/test_host.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -637,6 +637,27 @@ def test_get_capabilities_no_host_cpu_model(self):
637637
self.assertIsNone(caps.host.cpu.model)
638638
self.assertEqual(0, len(caps.host.cpu.features))
639639

640+
def _test_get_domain_capabilities(self):
641+
caps = self.host.get_domain_capabilities()
642+
self.assertIn('x86_64', caps.keys())
643+
self.assertEqual(['q35'], list(caps['x86_64']))
644+
return caps['x86_64']['q35']
645+
646+
def test_get_domain_capabilities(self):
647+
caps = self._test_get_domain_capabilities()
648+
self.assertEqual(vconfig.LibvirtConfigDomainCaps, type(caps))
649+
# There is a <gic supported='no'/> feature in the fixture but
650+
# we don't parse that because nothing currently cares about it.
651+
self.assertEqual(0, len(caps.features))
652+
653+
@mock.patch.object(fakelibvirt.virConnect, '_domain_capability_features',
654+
new='')
655+
def test_get_domain_capabilities_no_features(self):
656+
caps = self._test_get_domain_capabilities()
657+
self.assertEqual(vconfig.LibvirtConfigDomainCaps, type(caps))
658+
features = caps.features
659+
self.assertEqual([], features)
660+
640661
@mock.patch.object(fakelibvirt.virConnect, "getHostname")
641662
def test_get_hostname_caching(self, mock_hostname):
642663
mock_hostname.return_value = "foo"

‎nova/virt/libvirt/config.py

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,59 @@ def format_dom(self):
112112
return caps
113113

114114

115+
class LibvirtConfigDomainCaps(LibvirtConfigObject):
116+
117+
def __init__(self, **kwargs):
118+
super(LibvirtConfigDomainCaps, self).__init__(
119+
root_name="domainCapabilities", **kwargs)
120+
self._features = None
121+
122+
def parse_dom(self, xmldoc):
123+
super(LibvirtConfigDomainCaps, self).parse_dom(xmldoc)
124+
125+
for c in xmldoc.getchildren():
126+
if c.tag == "features":
127+
features = LibvirtConfigDomainCapsFeatures()
128+
features.parse_dom(c)
129+
self._features = features
130+
131+
@property
132+
def features(self):
133+
if self._features is None:
134+
return []
135+
return self._features.features
136+
137+
138+
class LibvirtConfigDomainCapsFeatures(LibvirtConfigObject):
139+
140+
def __init__(self, **kwargs):
141+
super(LibvirtConfigDomainCapsFeatures, self).__init__(
142+
root_name="features", **kwargs)
143+
self.features = []
144+
145+
def parse_dom(self, xmldoc):
146+
super(LibvirtConfigDomainCapsFeatures, self).parse_dom(xmldoc)
147+
148+
for c in xmldoc.getchildren():
149+
feature = None
150+
# TODO(aspiers): add supported features here
151+
if feature:
152+
feature.parse_dom(c)
153+
self.features.append(feature)
154+
155+
# There are many other features and domain capabilities,
156+
# but we don't need to regenerate the XML (it's read-only
157+
# data provided by libvirtd), so there's no point parsing
158+
# them until we actually need their values.
159+
160+
# For the same reason, we do not need a format_dom() method, but
161+
# it's a bug if this ever gets called and we inherited one from
162+
# the base class, so override that to watch out for accidental
163+
# calls.
164+
def format_dom(self):
165+
raise RuntimeError(_('BUG: tried to generate domainCapabilities XML'))
166+
167+
115168
class LibvirtConfigCapsNUMATopology(LibvirtConfigObject):
116169

117170
def __init__(self, **kwargs):

‎nova/virt/libvirt/host.py

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
the other libvirt related classes
2828
"""
2929

30+
from collections import defaultdict
3031
import operator
3132
import os
3233
import socket
@@ -56,6 +57,7 @@
5657
from nova.virt import event as virtevent
5758
from nova.virt.libvirt import config as vconfig
5859
from nova.virt.libvirt import guest as libvirt_guest
60+
from nova.virt.libvirt import utils as libvirt_utils
5961

6062
libvirt = None
6163

@@ -91,6 +93,7 @@ def __init__(self, uri, read_only=False,
9193
self._conn_event_handler_queue = six.moves.queue.Queue()
9294
self._lifecycle_event_handler = lifecycle_event_handler
9395
self._caps = None
96+
self._domain_caps = None
9497
self._hostname = None
9598

9699
self._wrapped_conn = None
@@ -667,6 +670,117 @@ def get_capabilities(self):
667670
raise
668671
return self._caps
669672

673+
def get_domain_capabilities(self):
674+
"""Returns the capabilities you can request when creating a
675+
domain (VM) with that hypervisor, for various combinations of
676+
architecture and machine type.
677+
678+
In this context the fuzzy word "hypervisor" implies QEMU
679+
binary, libvirt itself and the host config. libvirt provides
680+
this in order that callers can determine what the underlying
681+
emulator and/or libvirt is capable of, prior to creating a domain
682+
(for instance via virDomainCreateXML or virDomainDefineXML).
683+
However nova needs to know the capabilities much earlier, when
684+
the host's compute service is first initialised, in order that
685+
placement decisions can be made across many compute hosts.
686+
Therefore this is expected to be called during the init_host()
687+
phase of the driver lifecycle rather than just before booting
688+
an instance.
689+
690+
This causes an additional complication since the Python
691+
binding for this libvirt API call requires the architecture
692+
and machine type to be provided. So in order to gain a full
693+
picture of the hypervisor's capabilities, technically we need
694+
to call it with the right parameters, once for each
695+
(architecture, machine_type) combination which we care about.
696+
However the libvirt experts have advised us that in practice
697+
the domain capabilities do not (yet, at least) vary enough
698+
across machine types to justify the cost of calling
699+
getDomainCapabilities() once for every single (architecture,
700+
machine_type) combination. In particular, SEV support isn't
701+
reported per-machine type, and since there are usually many
702+
machine types, we follow the advice of the experts that for
703+
now it's sufficient to call it once per host architecture:
704+
705+
https://bugzilla.redhat.com/show_bug.cgi?id=1683471#c7
706+
707+
However, future domain capabilities might report SEV in a more
708+
fine-grained manner, and we also expect to use this method to
709+
detect other features, such as for gracefully handling machine
710+
types and potentially for detecting OVMF binaries. Therefore
711+
we memoize the results of the API calls in a nested dict where
712+
the top-level keys are architectures, and second-level keys
713+
are machine types, in order to allow easy expansion later.
714+
715+
Whenever libvirt/QEMU are updated, cached domCapabilities
716+
would get outdated (because QEMU will contain new features and
717+
the capabilities will vary). However, this should not be a
718+
problem here, because when libvirt/QEMU gets updated, the
719+
nova-compute agent also needs restarting, at which point the
720+
memoization will vanish because it's not persisted to disk.
721+
722+
Note: The result is cached in the member attribute
723+
_domain_caps.
724+
725+
:returns: a nested dict of dicts which maps architectures to
726+
machine types to instances of config.LibvirtConfigDomainCaps
727+
representing the domain capabilities of the host for that arch
728+
and machine type:
729+
730+
{ arch:
731+
{ machine_type: LibvirtConfigDomainCaps }
732+
}
733+
"""
734+
if self._domain_caps:
735+
return self._domain_caps
736+
737+
domain_caps = defaultdict(dict)
738+
caps = self.get_capabilities()
739+
virt_type = CONF.libvirt.virt_type
740+
741+
for guest in caps.guests:
742+
arch = guest.arch
743+
machine_type = \
744+
libvirt_utils.get_default_machine_type(arch) or 'q35'
745+
746+
emulator_bin = guest.emulator
747+
if virt_type in guest.domemulator:
748+
emulator_bin = guest.domemulator[virt_type]
749+
750+
# It is expected that each <guest> will have a different
751+
# architecture, but it doesn't hurt to add a safety net to
752+
# avoid needlessly calling libvirt's API more times than
753+
# we need.
754+
if machine_type in domain_caps[arch]:
755+
continue
756+
757+
domain_caps[arch][machine_type] = \
758+
self._get_domain_capabilities(emulator_bin, arch,
759+
machine_type, virt_type)
760+
761+
# NOTE(aspiers): Use a temporary variable to update the
762+
# instance variable atomically, otherwise if some API
763+
# calls succeeded and then one failed, we might
764+
# accidentally memoize a partial result.
765+
self._domain_caps = domain_caps
766+
767+
return self._domain_caps
768+
769+
def _get_domain_capabilities(self, emulator_bin, arch, machine_type,
770+
virt_type, flags=0):
771+
xmlstr = self.get_connection().getDomainCapabilities(
772+
emulator_bin,
773+
arch,
774+
machine_type,
775+
virt_type,
776+
flags
777+
)
778+
LOG.info("Libvirt host hypervisor capabilities for arch=%s and "
779+
"machine_type=%s:\n%s", arch, machine_type, xmlstr)
780+
caps = vconfig.LibvirtConfigDomainCaps()
781+
caps.parse_str(xmlstr)
782+
return caps
783+
670784
def get_driver_type(self):
671785
"""Get hypervisor type.
672786

0 commit comments

Comments
 (0)