Merge "Cyborg evacuate support"

Author: Zuul

api-guide/source/accelerator-support.rst

@@ -29,9 +29,8 @@ appropriate privileges) must do the following:
 
     openstack server create --flavor $myflavor --image $myimage $servername
 
-As of 21.0.0 (Ussuri), nova supports only specific operations for instances
-with accelerators. The lists of supported and unsupported operations are as
-below:
+Nova supports only specific operations for instances with accelerators.
+The lists of supported and unsupported operations are as below:
 
 * Supported operations.
 
@@ -42,17 +41,21 @@ below:
   * Take a snapshot.
   * Backup.
   * Rescue and unrescue.
+  * Rebuild.
+  * Evacuate.
 
 * Unsupported operations
 
-  * Rebuild.
   * Resize.
-  * Evacuate.
   * Suspend and resume.
   * Shelve and unshelve.
   * Cold migration.
   * Live migration.
 
+.. versionchanged:: 22.0.0(Victoria)
+
+   Added support for rebuild and evacuate operations.
+
 Some operations, such as lock and unlock, work as they are effectively
 no-ops for accelerators.
 

nova/accelerator/cyborg.py

@@ -302,3 +302,12 @@ def delete_arqs_by_uuid(self, arq_uuids):
         if err_msg:
             # No point raising an exception.
             LOG.error('Failed to delete ARQs %s', arq_uuid_str)
+
+    def get_arq_uuids_for_instance(self, instance):
+        """Get ARQ UUIDs for the instance.
+
+        :param instance: Instance Object
+        :return: ARQ UUIDs.
+        """
+        return [arq['uuid']
+                for arq in self.get_arqs_for_instance(instance.uuid)]

nova/compute/api.py

@@ -109,6 +109,8 @@
 # TODO(huaqiang): Remove in Wallaby
 MIN_VER_NOVA_COMPUTE_MIXED_POLICY = 52
 
+SUPPORT_ACCELERATOR_SERVICE_FOR_REBUILD = 53
+
 # FIXME(danms): Keep a global cache of the cells we find the
 # first time we look. This needs to be refreshed on a timer or
 # trigger.
@@ -307,14 +309,27 @@ def _get_image_meta_obj(image_meta_dict):
     return image_meta
 
 
-def block_accelerators(func):
-    @functools.wraps(func)
-    def wrapper(self, context, instance, *args, **kwargs):
-        dp_name = instance.flavor.extra_specs.get('accel:device_profile')
-        if dp_name:
-            raise exception.ForbiddenWithAccelerators()
-        return func(self, context, instance, *args, **kwargs)
-    return wrapper
+def block_accelerators(until_service=None):
+    def inner(func):
+        @functools.wraps(func)
+        def wrapper(self, context, instance, *args, **kwargs):
+            # NOTE(brinzhang): Catch a request operating a mixed instance,
+            # make sure all nova-compute services have been upgraded and
+            # support the accelerators.
+            dp_name = instance.flavor.extra_specs.get('accel:device_profile')
+            service_support = False
+            if not dp_name:
+                service_support = True
+            elif until_service:
+                min_version = objects.service.get_minimum_version_all_cells(
+                    nova_context.get_admin_context(), ['nova-compute'])
+                if min_version >= until_service:
+                    service_support = True
+            if not service_support:
+                raise exception.ForbiddenWithAccelerators()
+            return func(self, context, instance, *args, **kwargs)
+        return wrapper
+    return inner
 
 
 @profiler.trace_cls("compute_api")
@@ -3393,7 +3408,7 @@ def _check_image_arch(self, image=None):
                 fields_obj.Architecture.canonicalize(img_arch)
 
     @reject_vtpm_instances(instance_actions.REBUILD)
-    @block_accelerators
+    @block_accelerators(until_service=SUPPORT_ACCELERATOR_SERVICE_FOR_REBUILD)
     # TODO(stephenfin): We should expand kwargs out to named args
     @check_instance_lock
     @check_instance_state(vm_state=[vm_states.ACTIVE, vm_states.STOPPED,
@@ -3930,7 +3945,7 @@ def _validate_host_for_cold_migrate(
     # finally split resize and cold migration into separate code paths
     # TODO(stephenfin): The 'block_accelerators' decorator doesn't take into
     # account the accelerators requested in the new flavor
-    @block_accelerators
+    @block_accelerators()
     @check_instance_lock
     @check_instance_state(vm_state=[vm_states.ACTIVE, vm_states.STOPPED])
     @check_instance_host(check_is_up=True)
@@ -4159,7 +4174,7 @@ def _allow_resize_to_same_host(self, cold_migrate, instance):
         return allow_same_host
 
     @reject_vtpm_instances(instance_actions.SHELVE)
-    @block_accelerators
+    @block_accelerators()
     @check_instance_lock
     @check_instance_state(vm_state=[vm_states.ACTIVE, vm_states.STOPPED,
                                     vm_states.PAUSED, vm_states.SUSPENDED])
@@ -4324,7 +4339,7 @@ def get_instance_diagnostics(self, context, instance):
         return self.compute_rpcapi.get_instance_diagnostics(context,
                                                             instance=instance)
 
-    @block_accelerators
+    @block_accelerators()
     @reject_sev_instances(instance_actions.SUSPEND)
     @check_instance_lock
     @check_instance_state(vm_state=[vm_states.ACTIVE])
@@ -5028,7 +5043,7 @@ def update_instance_metadata(self, context, instance,
                                                      diff=diff)
         return _metadata
 
-    @block_accelerators
+    @block_accelerators()
     @reject_vtpm_instances(instance_actions.LIVE_MIGRATION)
     @reject_sev_instances(instance_actions.LIVE_MIGRATION)
     @check_instance_lock
@@ -5160,7 +5175,7 @@ def live_migrate_abort(self, context, instance, migration_id,
                 instance, migration.id)
 
     @reject_vtpm_instances(instance_actions.EVACUATE)
-    @block_accelerators
+    @block_accelerators(until_service=SUPPORT_ACCELERATOR_SERVICE_FOR_REBUILD)
     @check_instance_state(vm_state=[vm_states.ACTIVE, vm_states.STOPPED,
                                     vm_states.ERROR])
     def evacuate(self, context, instance, host, on_shared_storage,

nova/compute/manager.py

@@ -530,7 +530,7 @@ def update_compute_provider_status(self, context, rp_uuid, enabled):
 class ComputeManager(manager.Manager):
     """Manages the running instances from creation to destruction."""
 
-    target = messaging.Target(version='5.11')
+    target = messaging.Target(version='5.12')
 
     def __init__(self, compute_driver=None, *args, **kwargs):
         """Load configuration options and connect to the hypervisor."""
@@ -3256,25 +3256,44 @@ def _set_migration_status(migration, status):
             migration.status = status
             migration.save()
 
-    def _rebuild_default_impl(self, context, instance, image_meta,
-                              injected_files, admin_password, allocations,
-                              bdms, detach_block_devices, attach_block_devices,
-                              network_info=None,
-                              evacuate=False, block_device_info=None,
-                              preserve_ephemeral=False):
+    def _rebuild_default_impl(
+            self, context, instance, image_meta, injected_files,
+            admin_password, allocations, bdms, detach_block_devices,
+            attach_block_devices, network_info=None, evacuate=False,
+            block_device_info=None, preserve_ephemeral=False,
+            accel_uuids=None):
         if preserve_ephemeral:
             # The default code path does not support preserving ephemeral
             # partitions.
             raise exception.PreserveEphemeralNotSupported()
 
+        accel_info = []
         if evacuate:
+            if instance.flavor.extra_specs.get('accel:device_profile'):
+                try:
+                    accel_info = self._get_bound_arq_resources(
+                        context, instance, accel_uuids or [])
+                except (Exception, eventlet.timeout.Timeout) as exc:
+                    LOG.exception(exc)
+                    self._build_resources_cleanup(instance, network_info)
+                    msg = _('Failure getting accelerator resources.')
+                    raise exception.BuildAbortException(
+                        instance_uuid=instance.uuid, reason=msg)
             detach_block_devices(context, bdms)
         else:
             self._power_off_instance(instance, clean_shutdown=True)
             detach_block_devices(context, bdms)
             self.driver.destroy(context, instance,
                                 network_info=network_info,
                                 block_device_info=block_device_info)
+            try:
+                accel_info = self._get_accel_info(context, instance)
+            except Exception as exc:
+                LOG.exception(exc)
+                self._build_resources_cleanup(instance, network_info)
+                msg = _('Failure getting accelerator resources.')
+                raise exception.BuildAbortException(
+                    instance_uuid=instance.uuid, reason=msg)
 
         instance.task_state = task_states.REBUILD_BLOCK_DEVICE_MAPPING
         instance.save(expected_task_state=[task_states.REBUILDING])
@@ -3289,7 +3308,8 @@ def _rebuild_default_impl(self, context, instance, image_meta,
             self.driver.spawn(context, instance, image_meta, injected_files,
                               admin_password, allocations,
                               network_info=network_info,
-                              block_device_info=new_block_device_info)
+                              block_device_info=new_block_device_info,
+                              accel_info=accel_info)
 
     def _notify_instance_rebuild_error(self, context, instance, error, bdms):
         self._notify_about_instance_usage(context, instance,
@@ -3298,7 +3318,8 @@ def _notify_instance_rebuild_error(self, context, instance, error, bdms):
             context, instance, self.host,
             phase=fields.NotificationPhase.ERROR, exception=error, bdms=bdms)
 
-    @messaging.expected_exceptions(exception.PreserveEphemeralNotSupported)
+    @messaging.expected_exceptions(exception.PreserveEphemeralNotSupported,
+                                   exception.BuildAbortException)
     @wrap_exception()
     @reverts_task_state
     @wrap_instance_event(prefix='compute')
@@ -3307,7 +3328,7 @@ def rebuild_instance(self, context, instance, orig_image_ref, image_ref,
                          injected_files, new_pass, orig_sys_metadata,
                          bdms, recreate, on_shared_storage,
                          preserve_ephemeral, migration,
-                         scheduled_node, limits, request_spec):
+                         scheduled_node, limits, request_spec, accel_uuids):
         """Destroy and re-make this instance.
 
         A 'rebuild' effectively purges all existing data from the system and
@@ -3338,6 +3359,7 @@ def rebuild_instance(self, context, instance, orig_image_ref, image_ref,
         :param limits: Overcommit limits set by the scheduler. If a host was
                        specified by the user, this will be None
         :param request_spec: a RequestSpec object used to schedule the instance
+        :param accel_uuids: a list of cyborg ARQ uuids.
 
         """
         # recreate=True means the instance is being evacuated from a failed
@@ -3402,7 +3424,7 @@ def rebuild_instance(self, context, instance, orig_image_ref, image_ref,
                     image_meta, injected_files, new_pass, orig_sys_metadata,
                     bdms, evacuate, on_shared_storage, preserve_ephemeral,
                     migration, request_spec, allocs, rebuild_claim,
-                    scheduled_node, limits)
+                    scheduled_node, limits, accel_uuids)
             except (exception.ComputeResourcesUnavailable,
                     exception.RescheduledException) as e:
                 if isinstance(e, exception.ComputeResourcesUnavailable):
@@ -3469,7 +3491,7 @@ def _do_rebuild_instance_with_claim(
             self, context, instance, orig_image_ref, image_meta,
             injected_files, new_pass, orig_sys_metadata, bdms, evacuate,
             on_shared_storage, preserve_ephemeral, migration, request_spec,
-            allocations, rebuild_claim, scheduled_node, limits):
+            allocations, rebuild_claim, scheduled_node, limits, accel_uuids):
         """Helper to avoid deep nesting in the top-level method."""
 
         provider_mapping = None
@@ -3490,7 +3512,7 @@ def _do_rebuild_instance_with_claim(
                 context, instance, orig_image_ref, image_meta, injected_files,
                 new_pass, orig_sys_metadata, bdms, evacuate, on_shared_storage,
                 preserve_ephemeral, migration, request_spec, allocations,
-                provider_mapping)
+                provider_mapping, accel_uuids)
 
     @staticmethod
     def _get_image_name(image_meta):
@@ -3499,12 +3521,12 @@ def _get_image_name(image_meta):
         else:
             return ''
 
-    def _do_rebuild_instance(self, context, instance, orig_image_ref,
-                             image_meta, injected_files, new_pass,
-                             orig_sys_metadata, bdms, evacuate,
-                             on_shared_storage, preserve_ephemeral,
-                             migration, request_spec, allocations,
-                             request_group_resource_providers_mapping):
+    def _do_rebuild_instance(
+            self, context, instance, orig_image_ref, image_meta,
+            injected_files, new_pass, orig_sys_metadata, bdms, evacuate,
+            on_shared_storage, preserve_ephemeral, migration, request_spec,
+            allocations, request_group_resource_providers_mapping,
+            accel_uuids):
         orig_vm_state = instance.vm_state
 
         if evacuate:
@@ -3645,7 +3667,8 @@ def detach_block_devices(context, bdms):
             block_device_info=block_device_info,
             network_info=network_info,
             preserve_ephemeral=preserve_ephemeral,
-            evacuate=evacuate)
+            evacuate=evacuate,
+            accel_uuids=accel_uuids)
         try:
             with instance.mutated_migration_context():
                 self.driver.rebuild(**kwargs)

nova/compute/rpcapi.py

@@ -378,6 +378,8 @@ class ComputeAPI(object):
         * 5.10 - Add finish_revert_snapshot_based_resize_at_source()
         * 5.11 - Add accel_uuids (accelerator requests) parameter to
                  build_and_run_instance()
+        * 5.12 - Add accel_uuids (accelerator requests) parameter to
+                 rebuild_instance()
     '''
 
     VERSION_ALIASES = {
@@ -1056,20 +1058,29 @@ def reboot_instance(self, ctxt, instance, block_device_info,
                    block_device_info=block_device_info,
                    reboot_type=reboot_type)
 
-    def rebuild_instance(self, ctxt, instance, new_pass, injected_files,
+    def rebuild_instance(
+            self, ctxt, instance, new_pass, injected_files,
             image_ref, orig_image_ref, orig_sys_metadata, bdms,
             recreate, on_shared_storage, host, node,
-            preserve_ephemeral, migration, limits, request_spec):
+            preserve_ephemeral, migration, limits, request_spec, accel_uuids):
+
         # NOTE(edleafe): compute nodes can only use the dict form of limits.
         if isinstance(limits, objects.SchedulerLimits):
             limits = limits.to_dict()
-        msg_args = {'preserve_ephemeral': preserve_ephemeral,
-                    'migration': migration,
-                    'scheduled_node': node,
-                    'limits': limits,
-                    'request_spec': request_spec}
-        version = '5.0'
+
+        msg_args = {
+            'preserve_ephemeral': preserve_ephemeral,
+            'migration': migration,
+            'scheduled_node': node,
+            'limits': limits,
+            'request_spec': request_spec,
+            'accel_uuids': accel_uuids
+        }
+        version = '5.12'
         client = self.router.client(ctxt)
+        if not client.can_send_version(version):
+            del msg_args['accel_uuids']
+            version = '5.0'
         cctxt = client.prepare(server=_compute_host(host, instance),
                 version=version)
         cctxt.cast(ctxt, 'rebuild_instance',