SR-IOV Live migration indirect port support

This patch, builds on previous patches and enables
Live migration with SR-IOV indirect ports.

Prior to this change migration would have either:
  - Failed with instance running on the source node.
  - Failed with two VMs booted on both source and destination
    nodes, VM state is set to migrating, duplicate MACs on
    source and destination node and improper PCI resource claiming.
    This scenario is observed in the case of macvtap port type and
    neutron does not support multiple port binding API extension.
    With very little, non user friendly information in the log.

Conductor Changes:
- Allow live migration only with VIF related PCI devices to allow
  properly claiming PCI resources on the destination node.
  With this change live migration with generic flavor based
  PCI passthrough devices will not be supported due to libvirt and
  qemu constraint.

- Add a check to allow live migration with VIF related PCI allocation
  only when neutron supports multiple ports binding API extension
  and compute nodes are up to date.

- update the migrating VIF with the correct profile when binding the
  ports on the destination host, this will allow proper binding against
  the destination host and ensure VIF will be plugged correctly by Nova.

Compute Changes:
- Create VIFMigrateData for all VIFs in
  check_can_live_migrate_destination()

- For every VIF that contains a PCI device in its profile
  claim a PCI device on the destination node using the matching
  InstancePCIRequest of the instance being migrated.

- Update the relevant VIFMigrateData profile with the newly
  claimed PCI device.

- Free PCI devices on source and allocate on destination upon
  a successful migration or free claimed PCI devices on destination
  upon failure.

NeutronV2 Changes:
- Don't update binding profile with PCI devices if migration type is
  live-migration as the profile was already updated when an inactive
  port binding was created during bind_ports_to_host() call
  from conductor.

  Note: This builds on multiple ports binding API.

Change-Id: I734cc01dce13f9e75a16639faf890ddb1661b7eb
Partial-Implement: blueprint libvirt-neutron-sriov-livemigration

Author: adrianchiris

nova/compute/manager.py

@@ -28,6 +28,7 @@
 import base64
 import binascii
 import contextlib
+import copy
 import functools
 import inspect
 import sys
@@ -85,6 +86,7 @@
 from nova.objects import fields
 from nova.objects import instance as obj_instance
 from nova.objects import migrate_data as migrate_data_obj
+from nova.pci import request as pci_req_module
 from nova.pci import whitelist
 from nova import rpc
 from nova import safe_utils
@@ -6221,6 +6223,15 @@ def check_can_live_migrate_destination(self, ctxt, instance,
             migrate_data = self.compute_rpcapi.\
                                 check_can_live_migrate_source(ctxt, instance,
                                                               dest_check_data)
+            # Create migrate_data vifs
+            migrate_data.vifs = \
+                migrate_data_obj.LiveMigrateData.create_skeleton_migrate_vifs(
+                    instance.get_network_info())
+            # Claim PCI devices for VIFs on destination (if needed)
+            port_id_to_pci = self._claim_pci_for_instance_vifs(ctxt, instance)
+            # Update migrate VIFs with the newly claimed PCI devices
+            self._update_migrate_vifs_profile_with_pci(migrate_data.vifs,
+                                                       port_id_to_pci)
         finally:
             self.driver.cleanup_live_migration_destination_check(ctxt,
                     dest_check_data)
@@ -6853,6 +6864,9 @@ def _post_live_migration(self, ctxt, instance, dest,
             # method
             destroy_vifs = True
 
+        # Free instance allocations on source before claims are allocated on
+        # destination node
+        self.rt.free_pci_device_allocations_for_instance(ctxt, instance)
         # NOTE(danms): Save source node before calling post method on
         # destination, which will update it
         source_node = instance.node
@@ -6955,7 +6969,8 @@ def post_live_migration_at_destination(self, context, instance,
         self.network_api.setup_networks_on_host(context, instance,
                                                          self.host)
         migration = {'source_compute': instance.host,
-                     'dest_compute': self.host, }
+                     'dest_compute': self.host,
+                     'migration_type': 'live-migration'}
         self.network_api.migrate_instance_finish(context,
                                                  instance,
                                                  migration)
@@ -6970,6 +6985,8 @@ def post_live_migration_at_destination(self, context, instance,
                 phase=fields.NotificationPhase.START)
         block_device_info = self._get_instance_block_device_info(context,
                                                                  instance)
+        # Allocate the claimed PCI resources at destination.
+        self.rt.allocate_pci_devices_for_instance(context, instance)
 
         try:
             self.driver.post_live_migration_at_destination(
@@ -7187,6 +7204,10 @@ def rollback_live_migration_at_destination(self, context, instance,
             #             from remote volumes if necessary
             block_device_info = self._get_instance_block_device_info(context,
                                                                      instance)
+            # free any instance PCI claims done on destination during
+            # check_can_live_migrate_destination()
+            self.rt.free_pci_device_claims_for_instance(context, instance)
+
             self.driver.rollback_live_migration_at_destination(
                 context, instance, network_info, block_device_info,
                 destroy_disks=destroy_disks, migrate_data=migrate_data)
@@ -8532,3 +8553,70 @@ def _cleanup_expired_console_auth_tokens(self, context):
         """
         objects.ConsoleAuthToken.clean_expired_console_auths_for_host(
             context, self.host)
+
+    def _claim_pci_for_instance_vifs(self, ctxt, instance):
+        """Claim PCI devices for the instance's VIFs on the compute node
+
+        :param ctxt: Context
+        :param instance: Instance object
+        :return: <port ID: PciDevice> mapping for the VIFs that yielded a
+                PCI claim on the compute node
+        """
+        pci_req_id_to_port_id = {}
+        pci_reqs = []
+        port_id_to_pci_dev = {}
+
+        for vif in instance.get_network_info():
+            pci_req = pci_req_module.get_instance_pci_request_from_vif(
+                ctxt,
+                instance,
+                vif)
+            if pci_req:
+                pci_req_id_to_port_id[pci_req.request_id] = vif['id']
+                pci_reqs.append(pci_req)
+
+        if pci_reqs:
+            # Create PCI requests and claim against PCI resource tracker
+            # NOTE(adrianc): We claim against the same requests as on the
+            # source node.
+            vif_pci_requests = objects.InstancePCIRequests(
+                requests=pci_reqs,
+                instance_uuid=instance.uuid)
+
+            claimed_pci_devices_objs = self.rt.claim_pci_devices(
+                ctxt,
+                vif_pci_requests)
+
+            # Update VIFMigrateData profile with the newly claimed PCI
+            # device
+            for pci_dev in claimed_pci_devices_objs:
+                LOG.debug("PCI device: %s Claimed on destination node",
+                          pci_dev.address)
+                port_id = pci_req_id_to_port_id[pci_dev.request_id]
+                port_id_to_pci_dev[port_id] = pci_dev
+
+        return port_id_to_pci_dev
+
+    def _update_migrate_vifs_profile_with_pci(self,
+                                              migrate_vifs,
+                                              port_id_to_pci_dev):
+        """Update migrate vifs profile with the claimed PCI devices
+
+        :param migrate_vifs: list of VIFMigrateData objects
+        :param port_id_to_pci_dev: a <port_id: PciDevice> mapping
+        :return: None.
+        """
+        for mig_vif in migrate_vifs:
+            port_id = mig_vif.port_id
+            if port_id not in port_id_to_pci_dev:
+                continue
+
+            pci_dev = port_id_to_pci_dev[port_id]
+            profile = copy.deepcopy(mig_vif.source_vif['profile'])
+            profile['pci_slot'] = pci_dev.address
+            profile['pci_vendor_info'] = ':'.join([pci_dev.vendor_id,
+                                                   pci_dev.product_id])
+            mig_vif.profile = profile
+            LOG.debug("Updating migrate VIF profile for port %(port_id)s:"
+                      "%(profile)s", {'port_id': port_id,
+                                      'profile': profile})

nova/compute/resource_tracker.py

@@ -1514,3 +1514,46 @@ def build_failed(self, nodename):
     def build_succeeded(self, nodename):
         """Resets the failed_builds stats for the given node."""
         self.stats[nodename].build_succeeded()
+
+    @utils.synchronized(COMPUTE_RESOURCE_SEMAPHORE)
+    def claim_pci_devices(self, context, pci_requests):
+        """Claim instance PCI resources
+
+        :param context: security context
+        :param pci_requests: a list of nova.objects.InstancePCIRequests
+        :returns: a list of nova.objects.PciDevice objects
+        """
+        result = self.pci_tracker.claim_instance(
+            context, pci_requests, None)
+        self.pci_tracker.save(context)
+        return result
+
+    @utils.synchronized(COMPUTE_RESOURCE_SEMAPHORE)
+    def allocate_pci_devices_for_instance(self, context, instance):
+        """Allocate instance claimed PCI resources
+
+        :param context: security context
+        :param instance: instance object
+        """
+        self.pci_tracker.allocate_instance(instance)
+        self.pci_tracker.save(context)
+
+    @utils.synchronized(COMPUTE_RESOURCE_SEMAPHORE)
+    def free_pci_device_allocations_for_instance(self, context, instance):
+        """Free instance allocated PCI resources
+
+        :param context: security context
+        :param instance: instance object
+        """
+        self.pci_tracker.free_instance_allocations(context, instance)
+        self.pci_tracker.save(context)
+
+    @utils.synchronized(COMPUTE_RESOURCE_SEMAPHORE)
+    def free_pci_device_claims_for_instance(self, context, instance):
+        """Free instance claimed PCI resources
+
+        :param context: security context
+        :param instance: instance object
+        """
+        self.pci_tracker.free_instance_claims(context, instance)
+        self.pci_tracker.save(context)

nova/conductor/tasks/live_migrate.py

@@ -25,6 +25,7 @@
 from nova import network
 from nova import objects
 from nova.objects import fields as obj_fields
+from nova.objects import migrate_data as migrate_data_obj
 from nova.scheduler import utils as scheduler_utils
 
 LOG = logging.getLogger(__name__)
@@ -44,6 +45,19 @@ def supports_extended_port_binding(context, host):
     return svc.version >= 35
 
 
+def supports_vif_related_pci_allocations(context, host):
+    """Checks if the compute host service is new enough to support
+    VIF related PCI allocation during live migration
+
+    :param context: The user request context.
+    :param host: The nova-compute host to check.
+    :returns: True if the compute host is new enough to support vif related
+              PCI allocations
+    """
+    svc = objects.Service.get_by_host_and_binary(context, host, 'nova-compute')
+    return svc.version >= 36
+
+
 class LiveMigrationTask(base.TaskBase):
     def __init__(self, context, instance, destination,
                  block_migration, disk_over_commit, migration, compute_rpcapi,
@@ -186,6 +200,47 @@ def _check_instance_has_no_numa(self):
         else:
             raise exception.MigrationPreCheckError(reason=msg)
 
+    def _check_can_migrate_pci(self, src_host, dest_host):
+        """Checks that an instance can migrate with PCI requests.
+
+        At the moment support only if:
+
+            1. Instance contains VIF related PCI requests.
+            2. Neutron supports multiple port binding extension.
+            3. Src and Dest host support VIF related PCI allocations.
+        """
+        if self.instance.pci_requests is None or not len(
+                self.instance.pci_requests.requests):
+            return
+
+        for pci_request in self.instance.pci_requests.requests:
+            if pci_request.alias_name is not None:
+                # allow only VIF related PCI requests in live migration.
+                # PCI requests come from two sources: instance flavor and
+                # SR-IOV ports.
+                # SR-IOV ports pci_request don't have an alias_name.
+                # TODO(adrianc): add an is_sriov_port property to PCIRequest
+                # to make this cryptic check clearer (also in resource_tracker)
+
+                raise exception.MigrationPreCheckError(
+                    reason= "non-VIF related PCI requests for instance "
+                            "are not allowed for live migration.")
+        # All PCI requests are VIF related, now check neutron,
+        # source and destination compute nodes.
+        if not self.network_api.supports_port_binding_extension(
+                self.context):
+            raise exception.MigrationPreCheckError(
+                reason="Cannot live migrate VIF with related PCI, Neutron "
+                       "does not support required port binding extension.")
+        if not (supports_vif_related_pci_allocations(self.context,
+                                                     src_host) and
+                supports_vif_related_pci_allocations(self.context,
+                                                     dest_host)):
+            raise exception.MigrationPreCheckError(
+                reason="Cannot live migrate VIF with related PCI, "
+                       "source and destination nodes do not support "
+                       "the operation.")
+
     def _check_host_is_up(self, host):
         service = objects.Service.get_by_compute_host(self.context, host)
 
@@ -265,6 +320,7 @@ def _check_compatible_with_source_hypervisor(self, destination):
         return source_info, destination_info
 
     def _call_livem_checks_on_host(self, destination):
+        self._check_can_migrate_pci(self.source, destination)
         try:
             self.migrate_data = self.compute_rpcapi.\
                 check_can_live_migrate_destination(self.context, self.instance,
@@ -280,8 +336,17 @@ def _call_livem_checks_on_host(self, destination):
         if (self.network_api.supports_port_binding_extension(self.context) and
                 supports_extended_port_binding(self.context, self.source) and
                 supports_extended_port_binding(self.context, destination)):
-            self.migrate_data.vifs = (
-                self._bind_ports_on_destination(destination))
+            if 'vifs' not in self.migrate_data:
+                # migrate data vifs were not constructed in dest compute
+                # during check_can_live_migrate_destination, construct a
+                # skeleton to be updated after port binding.
+                # TODO(adrianc): This can be removed once we move to T release
+                self.migrate_data.vifs = migrate_data_obj.LiveMigrateData.\
+                    create_skeleton_migrate_vifs(
+                    self.instance.get_network_info())
+            bindings = self._bind_ports_on_destination(destination)
+            self._update_migrate_vifs_from_bindings(self.migrate_data.vifs,
+                                                    bindings)
 
     def _bind_ports_on_destination(self, destination):
         LOG.debug('Start binding ports on destination host: %s', destination,
@@ -291,23 +356,33 @@ def _bind_ports_on_destination(self, destination):
         # that was bound. This information is then stuffed into the
         # migrate_data.
         try:
+            # Note(adrianc): migrate_data.vifs was partially filled
+            # by destination compute if compute is new enough.
+            # if that is the case, it may have updated the required port
+            # profile for the destination node (e.g new PCI address if SR-IOV)
+            # perform port binding against the requested profile
+            migrate_vifs_with_profile = [mig_vif for mig_vif in
+                                         self.migrate_data.vifs
+                                         if 'profile_json' in mig_vif]
+
+            ports_profile = None
+            if migrate_vifs_with_profile:
+                # Update to the port profile is required
+                ports_profile = {mig_vif.port_id: mig_vif.profile
+                                 for mig_vif in migrate_vifs_with_profile}
+
             bindings = self.network_api.bind_ports_to_host(
-                self.context, self.instance, destination)
+                self.context, self.instance, destination, None, ports_profile)
         except exception.PortBindingFailed as e:
             # Port binding failed for that host, try another one.
             raise exception.MigrationPreCheckError(
                 reason=e.format_message())
+        return bindings
 
-        source_vif_map = {
-            vif['id']: vif for vif in self.instance.get_network_info()
-        }
-        migrate_vifs = []
-        for port_id, binding in bindings.items():
-            migrate_vif = objects.VIFMigrateData(
-                port_id=port_id, **binding)
-            migrate_vif.source_vif = source_vif_map[port_id]
-            migrate_vifs.append(migrate_vif)
-        return migrate_vifs
+    def _update_migrate_vifs_from_bindings(self, migrate_vifs, bindings):
+        for migrate_vif in migrate_vifs:
+            for attr_name, attr_val in bindings[migrate_vif.port_id].items():
+                setattr(migrate_vif, attr_name, attr_val)
 
     def _get_source_cell_mapping(self):
         """Returns the CellMapping for the cell in which the instance lives

nova/network/neutronv2/api.py

@@ -3335,7 +3335,11 @@ def _update_port_binding_for_instance(self, context, instance, host,
             # as in an unshelve operation.
             vnic_type = p.get('binding:vnic_type')
             if (vnic_type in network_model.VNIC_TYPES_SRIOV
-                    and migration is not None):
+                    and migration is not None
+                    and migration['migration_type'] !=
+                    constants.LIVE_MIGRATION):
+                # Note(adrianc): for live migration binding profile was already
+                # updated in conductor when calling bind_ports_to_host()
                 if not pci_mapping:
                     pci_mapping = self._get_pci_mapping_for_migration(context,
                         instance, migration)

nova/network/neutronv2/constants.py

@@ -20,3 +20,4 @@
 MULTI_NET_EXT = 'Multi Provider Network'
 SUBSTR_PORT_FILTERING = 'IP address substring filtering'
 PORT_BINDING_EXTENDED = 'Port Bindings Extended'
+LIVE_MIGRATION = 'live-migration'

nova/objects/migrate_data.py

@@ -120,6 +120,23 @@ class LiveMigrateData(obj_base.NovaObject):
         'vifs': fields.ListOfObjectsField('VIFMigrateData'),
     }
 
+    @staticmethod
+    def create_skeleton_migrate_vifs(vifs):
+        """Create migrate vifs for live migration.
+
+        :param vifs: a list of VIFs.
+        :return: list of VIFMigrateData object corresponding to the provided
+                 VIFs.
+        """
+        vif_mig_data = []
+
+        for vif in vifs:
+            mig_vif = VIFMigrateData(
+                port_id=vif['id'],
+                source_vif=vif)
+            vif_mig_data.append(mig_vif)
+        return vif_mig_data
+
 
 @obj_base.NovaObjectRegistry.register
 class LibvirtLiveMigrateBDMInfo(obj_base.NovaObject):