Merge "SR-IOV Live migration indirect port support"

Author: Zuul

nova/compute/manager.py

@@ -28,6 +28,7 @@
 import base64
 import binascii
 import contextlib
+import copy
 import functools
 import inspect
 import sys
@@ -85,6 +86,7 @@
 from nova.objects import fields
 from nova.objects import instance as obj_instance
 from nova.objects import migrate_data as migrate_data_obj
+from nova.pci import request as pci_req_module
 from nova.pci import whitelist
 from nova import rpc
 from nova import safe_utils
@@ -6221,6 +6223,15 @@ def check_can_live_migrate_destination(self, ctxt, instance,
             migrate_data = self.compute_rpcapi.\
                                 check_can_live_migrate_source(ctxt, instance,
                                                               dest_check_data)
+            # Create migrate_data vifs
+            migrate_data.vifs = \
+                migrate_data_obj.LiveMigrateData.create_skeleton_migrate_vifs(
+                    instance.get_network_info())
+            # Claim PCI devices for VIFs on destination (if needed)
+            port_id_to_pci = self._claim_pci_for_instance_vifs(ctxt, instance)
+            # Update migrate VIFs with the newly claimed PCI devices
+            self._update_migrate_vifs_profile_with_pci(migrate_data.vifs,
+                                                       port_id_to_pci)
         finally:
             self.driver.cleanup_live_migration_destination_check(ctxt,
                     dest_check_data)
@@ -6853,6 +6864,9 @@ def _post_live_migration(self, ctxt, instance, dest,
             # method
             destroy_vifs = True
 
+        # Free instance allocations on source before claims are allocated on
+        # destination node
+        self.rt.free_pci_device_allocations_for_instance(ctxt, instance)
         # NOTE(danms): Save source node before calling post method on
         # destination, which will update it
         source_node = instance.node
@@ -6955,7 +6969,8 @@ def post_live_migration_at_destination(self, context, instance,
         self.network_api.setup_networks_on_host(context, instance,
                                                          self.host)
         migration = {'source_compute': instance.host,
-                     'dest_compute': self.host, }
+                     'dest_compute': self.host,
+                     'migration_type': 'live-migration'}
         self.network_api.migrate_instance_finish(context,
                                                  instance,
                                                  migration)
@@ -6970,6 +6985,8 @@ def post_live_migration_at_destination(self, context, instance,
                 phase=fields.NotificationPhase.START)
         block_device_info = self._get_instance_block_device_info(context,
                                                                  instance)
+        # Allocate the claimed PCI resources at destination.
+        self.rt.allocate_pci_devices_for_instance(context, instance)
 
         try:
             self.driver.post_live_migration_at_destination(
@@ -7187,6 +7204,10 @@ def rollback_live_migration_at_destination(self, context, instance,
             #             from remote volumes if necessary
             block_device_info = self._get_instance_block_device_info(context,
                                                                      instance)
+            # free any instance PCI claims done on destination during
+            # check_can_live_migrate_destination()
+            self.rt.free_pci_device_claims_for_instance(context, instance)
+
             self.driver.rollback_live_migration_at_destination(
                 context, instance, network_info, block_device_info,
                 destroy_disks=destroy_disks, migrate_data=migrate_data)
@@ -8532,3 +8553,70 @@ def _cleanup_expired_console_auth_tokens(self, context):
         """
         objects.ConsoleAuthToken.clean_expired_console_auths_for_host(
             context, self.host)
+
+    def _claim_pci_for_instance_vifs(self, ctxt, instance):
+        """Claim PCI devices for the instance's VIFs on the compute node
+
+        :param ctxt: Context
+        :param instance: Instance object
+        :return: <port ID: PciDevice> mapping for the VIFs that yielded a
+                PCI claim on the compute node
+        """
+        pci_req_id_to_port_id = {}
+        pci_reqs = []
+        port_id_to_pci_dev = {}
+
+        for vif in instance.get_network_info():
+            pci_req = pci_req_module.get_instance_pci_request_from_vif(
+                ctxt,
+                instance,
+                vif)
+            if pci_req:
+                pci_req_id_to_port_id[pci_req.request_id] = vif['id']
+                pci_reqs.append(pci_req)
+
+        if pci_reqs:
+            # Create PCI requests and claim against PCI resource tracker
+            # NOTE(adrianc): We claim against the same requests as on the
+            # source node.
+            vif_pci_requests = objects.InstancePCIRequests(
+                requests=pci_reqs,
+                instance_uuid=instance.uuid)
+
+            claimed_pci_devices_objs = self.rt.claim_pci_devices(
+                ctxt,
+                vif_pci_requests)
+
+            # Update VIFMigrateData profile with the newly claimed PCI
+            # device
+            for pci_dev in claimed_pci_devices_objs:
+                LOG.debug("PCI device: %s Claimed on destination node",
+                          pci_dev.address)
+                port_id = pci_req_id_to_port_id[pci_dev.request_id]
+                port_id_to_pci_dev[port_id] = pci_dev
+
+        return port_id_to_pci_dev
+
+    def _update_migrate_vifs_profile_with_pci(self,
+                                              migrate_vifs,
+                                              port_id_to_pci_dev):
+        """Update migrate vifs profile with the claimed PCI devices
+
+        :param migrate_vifs: list of VIFMigrateData objects
+        :param port_id_to_pci_dev: a <port_id: PciDevice> mapping
+        :return: None.
+        """
+        for mig_vif in migrate_vifs:
+            port_id = mig_vif.port_id
+            if port_id not in port_id_to_pci_dev:
+                continue
+
+            pci_dev = port_id_to_pci_dev[port_id]
+            profile = copy.deepcopy(mig_vif.source_vif['profile'])
+            profile['pci_slot'] = pci_dev.address
+            profile['pci_vendor_info'] = ':'.join([pci_dev.vendor_id,
+                                                   pci_dev.product_id])
+            mig_vif.profile = profile
+            LOG.debug("Updating migrate VIF profile for port %(port_id)s:"
+                      "%(profile)s", {'port_id': port_id,
+                                      'profile': profile})

nova/compute/resource_tracker.py

@@ -1514,3 +1514,46 @@ def build_failed(self, nodename):
     def build_succeeded(self, nodename):
         """Resets the failed_builds stats for the given node."""
         self.stats[nodename].build_succeeded()
+
+    @utils.synchronized(COMPUTE_RESOURCE_SEMAPHORE)
+    def claim_pci_devices(self, context, pci_requests):
+        """Claim instance PCI resources
+
+        :param context: security context
+        :param pci_requests: a list of nova.objects.InstancePCIRequests
+        :returns: a list of nova.objects.PciDevice objects
+        """
+        result = self.pci_tracker.claim_instance(
+            context, pci_requests, None)
+        self.pci_tracker.save(context)
+        return result
+
+    @utils.synchronized(COMPUTE_RESOURCE_SEMAPHORE)
+    def allocate_pci_devices_for_instance(self, context, instance):
+        """Allocate instance claimed PCI resources
+
+        :param context: security context
+        :param instance: instance object
+        """
+        self.pci_tracker.allocate_instance(instance)
+        self.pci_tracker.save(context)
+
+    @utils.synchronized(COMPUTE_RESOURCE_SEMAPHORE)
+    def free_pci_device_allocations_for_instance(self, context, instance):
+        """Free instance allocated PCI resources
+
+        :param context: security context
+        :param instance: instance object
+        """
+        self.pci_tracker.free_instance_allocations(context, instance)
+        self.pci_tracker.save(context)
+
+    @utils.synchronized(COMPUTE_RESOURCE_SEMAPHORE)
+    def free_pci_device_claims_for_instance(self, context, instance):
+        """Free instance claimed PCI resources
+
+        :param context: security context
+        :param instance: instance object
+        """
+        self.pci_tracker.free_instance_claims(context, instance)
+        self.pci_tracker.save(context)

nova/conductor/tasks/live_migrate.py

@@ -25,6 +25,7 @@
 from nova import network
 from nova import objects
 from nova.objects import fields as obj_fields
+from nova.objects import migrate_data as migrate_data_obj
 from nova.scheduler import utils as scheduler_utils
 
 LOG = logging.getLogger(__name__)
@@ -44,6 +45,19 @@ def supports_extended_port_binding(context, host):
     return svc.version >= 35
 
 
+def supports_vif_related_pci_allocations(context, host):
+    """Checks if the compute host service is new enough to support
+    VIF related PCI allocation during live migration
+
+    :param context: The user request context.
+    :param host: The nova-compute host to check.
+    :returns: True if the compute host is new enough to support vif related
+              PCI allocations
+    """
+    svc = objects.Service.get_by_host_and_binary(context, host, 'nova-compute')
+    return svc.version >= 36
+
+
 class LiveMigrationTask(base.TaskBase):
     def __init__(self, context, instance, destination,
                  block_migration, disk_over_commit, migration, compute_rpcapi,
@@ -186,6 +200,47 @@ def _check_instance_has_no_numa(self):
         else:
             raise exception.MigrationPreCheckError(reason=msg)
 
+    def _check_can_migrate_pci(self, src_host, dest_host):
+        """Checks that an instance can migrate with PCI requests.
+
+        At the moment support only if:
+
+            1. Instance contains VIF related PCI requests.
+            2. Neutron supports multiple port binding extension.
+            3. Src and Dest host support VIF related PCI allocations.
+        """
+        if self.instance.pci_requests is None or not len(
+                self.instance.pci_requests.requests):
+            return
+
+        for pci_request in self.instance.pci_requests.requests:
+            if pci_request.alias_name is not None:
+                # allow only VIF related PCI requests in live migration.
+                # PCI requests come from two sources: instance flavor and
+                # SR-IOV ports.
+                # SR-IOV ports pci_request don't have an alias_name.
+                # TODO(adrianc): add an is_sriov_port property to PCIRequest
+                # to make this cryptic check clearer (also in resource_tracker)
+
+                raise exception.MigrationPreCheckError(
+                    reason= "non-VIF related PCI requests for instance "
+                            "are not allowed for live migration.")
+        # All PCI requests are VIF related, now check neutron,
+        # source and destination compute nodes.
+        if not self.network_api.supports_port_binding_extension(
+                self.context):
+            raise exception.MigrationPreCheckError(
+                reason="Cannot live migrate VIF with related PCI, Neutron "
+                       "does not support required port binding extension.")
+        if not (supports_vif_related_pci_allocations(self.context,
+                                                     src_host) and
+                supports_vif_related_pci_allocations(self.context,
+                                                     dest_host)):
+            raise exception.MigrationPreCheckError(
+                reason="Cannot live migrate VIF with related PCI, "
+                       "source and destination nodes do not support "
+                       "the operation.")
+
     def _check_host_is_up(self, host):
         service = objects.Service.get_by_compute_host(self.context, host)
 
@@ -265,6 +320,7 @@ def _check_compatible_with_source_hypervisor(self, destination):
         return source_info, destination_info
 
     def _call_livem_checks_on_host(self, destination):
+        self._check_can_migrate_pci(self.source, destination)
         try:
             self.migrate_data = self.compute_rpcapi.\
                 check_can_live_migrate_destination(self.context, self.instance,
@@ -280,8 +336,17 @@ def _call_livem_checks_on_host(self, destination):
         if (self.network_api.supports_port_binding_extension(self.context) and
                 supports_extended_port_binding(self.context, self.source) and
                 supports_extended_port_binding(self.context, destination)):
-            self.migrate_data.vifs = (
-                self._bind_ports_on_destination(destination))
+            if 'vifs' not in self.migrate_data:
+                # migrate data vifs were not constructed in dest compute
+                # during check_can_live_migrate_destination, construct a
+                # skeleton to be updated after port binding.
+                # TODO(adrianc): This can be removed once we move to T release
+                self.migrate_data.vifs = migrate_data_obj.LiveMigrateData.\
+                    create_skeleton_migrate_vifs(
+                    self.instance.get_network_info())
+            bindings = self._bind_ports_on_destination(destination)
+            self._update_migrate_vifs_from_bindings(self.migrate_data.vifs,
+                                                    bindings)
 
     def _bind_ports_on_destination(self, destination):
         LOG.debug('Start binding ports on destination host: %s', destination,
@@ -291,23 +356,33 @@ def _bind_ports_on_destination(self, destination):
         # that was bound. This information is then stuffed into the
         # migrate_data.
         try:
+            # Note(adrianc): migrate_data.vifs was partially filled
+            # by destination compute if compute is new enough.
+            # if that is the case, it may have updated the required port
+            # profile for the destination node (e.g new PCI address if SR-IOV)
+            # perform port binding against the requested profile
+            migrate_vifs_with_profile = [mig_vif for mig_vif in
+                                         self.migrate_data.vifs
+                                         if 'profile_json' in mig_vif]
+
+            ports_profile = None
+            if migrate_vifs_with_profile:
+                # Update to the port profile is required
+                ports_profile = {mig_vif.port_id: mig_vif.profile
+                                 for mig_vif in migrate_vifs_with_profile}
+
             bindings = self.network_api.bind_ports_to_host(
-                self.context, self.instance, destination)
+                self.context, self.instance, destination, None, ports_profile)
         except exception.PortBindingFailed as e:
             # Port binding failed for that host, try another one.
             raise exception.MigrationPreCheckError(
                 reason=e.format_message())
+        return bindings
 
-        source_vif_map = {
-            vif['id']: vif for vif in self.instance.get_network_info()
-        }
-        migrate_vifs = []
-        for port_id, binding in bindings.items():
-            migrate_vif = objects.VIFMigrateData(
-                port_id=port_id, **binding)
-            migrate_vif.source_vif = source_vif_map[port_id]
-            migrate_vifs.append(migrate_vif)
-        return migrate_vifs
+    def _update_migrate_vifs_from_bindings(self, migrate_vifs, bindings):
+        for migrate_vif in migrate_vifs:
+            for attr_name, attr_val in bindings[migrate_vif.port_id].items():
+                setattr(migrate_vif, attr_name, attr_val)
 
     def _get_source_cell_mapping(self):
         """Returns the CellMapping for the cell in which the instance lives

nova/network/neutronv2/api.py

@@ -3335,7 +3335,11 @@ def _update_port_binding_for_instance(self, context, instance, host,
             # as in an unshelve operation.
             vnic_type = p.get('binding:vnic_type')
             if (vnic_type in network_model.VNIC_TYPES_SRIOV
-                    and migration is not None):
+                    and migration is not None
+                    and migration['migration_type'] !=
+                    constants.LIVE_MIGRATION):
+                # Note(adrianc): for live migration binding profile was already
+                # updated in conductor when calling bind_ports_to_host()
                 if not pci_mapping:
                     pci_mapping = self._get_pci_mapping_for_migration(context,
                         instance, migration)

nova/network/neutronv2/constants.py

@@ -20,3 +20,4 @@
 MULTI_NET_EXT = 'Multi Provider Network'
 SUBSTR_PORT_FILTERING = 'IP address substring filtering'
 PORT_BINDING_EXTENDED = 'Port Bindings Extended'
+LIVE_MIGRATION = 'live-migration'

nova/objects/migrate_data.py

@@ -120,6 +120,23 @@ class LiveMigrateData(obj_base.NovaObject):
         'vifs': fields.ListOfObjectsField('VIFMigrateData'),
     }
 
+    @staticmethod
+    def create_skeleton_migrate_vifs(vifs):
+        """Create migrate vifs for live migration.
+
+        :param vifs: a list of VIFs.
+        :return: list of VIFMigrateData object corresponding to the provided
+                 VIFs.
+        """
+        vif_mig_data = []
+
+        for vif in vifs:
+            mig_vif = VIFMigrateData(
+                port_id=vif['id'],
+                source_vif=vif)
+            vif_mig_data.append(mig_vif)
+        return vif_mig_data
+
 
 @obj_base.NovaObjectRegistry.register
 class LibvirtLiveMigrateBDMInfo(obj_base.NovaObject):