Add finish_revert_snapshot_based_resize_at_source compute method

This adds the finish_revert_snapshot_based_resize_at_source() method
to the compute service. This completes the cross-cell resize revert
on the source host by updating port bindings, re-connecting volumes
and spawning the guest and waiting for the network-vif-plugged
event from neutron.

If the resize started with a stopped instance, the revert ensures
the guest is powered off when it is spawned.

The instance record is updated with the old_flavor information,
allocations are reverted in placement, and the migration status in
the source cell database is updated to "reverted".

Notifications are not sent from this method like in the
finish_revert_resize() method because those will be sent from
conductor.

Note that this implementation tries to be a bit more graceful
about error handling than the old same-cell finish_revert_resize
method since there are a lot of moving parts and we want to try
to clean up as much as possible.

Part of blueprint cross-cell-resize

Change-Id: I7e7afddbd9e0f57dfb1175a0bb2b54f2ed5500f2

Author: mriedem

nova/compute/manager.py

@@ -560,7 +560,7 @@ def update_compute_provider_status(self, context, rp_uuid, enabled):
 class ComputeManager(manager.Manager):
     """Manages the running instances from creation to destruction."""
 
-    target = messaging.Target(version='5.9')
+    target = messaging.Target(version='5.10')
 
     def __init__(self, compute_driver=None, *args, **kwargs):
         """Load configuration options and connect to the hypervisor."""
@@ -4686,6 +4686,153 @@ def _revert_snapshot_based_resize_at_dest(
         self.rt.drop_move_claim(ctxt, instance, instance.node,
                                 instance_type=instance.new_flavor)
 
+    @wrap_exception()
+    @reverts_task_state
+    @wrap_instance_event(prefix='compute')
+    @errors_out_migration
+    @wrap_instance_fault
+    def finish_revert_snapshot_based_resize_at_source(
+            self, ctxt, instance, migration):
+        """Reverts a snapshot-based resize at the source host.
+
+        Spawn the guest and re-connect volumes/VIFs on the source host and
+        revert the instance to use the old_flavor for resource usage reporting.
+
+        Updates allocations in the placement service to move the source node
+        allocations, held by the migration record, to the instance and drop
+        the allocations held by the instance on the destination node.
+
+        :param ctxt: nova auth request context targeted at the target cell
+        :param instance: Instance object whose vm_state is "resized" and
+            task_state is "resize_reverting".
+        :param migration: Migration object whose status is "reverting".
+        """
+
+        @utils.synchronized(instance.uuid)
+        def do_revert():
+            LOG.info('Reverting resize on source host.', instance=instance)
+            with self._error_out_instance_on_exception(ctxt, instance):
+                self._finish_revert_snapshot_based_resize_at_source(
+                    ctxt, instance, migration)
+        do_revert()
+
+        # Broadcast to all schedulers that the instance is on this host.
+        # This is best effort so if anything fails just log it.
+        try:
+            self._update_scheduler_instance_info(ctxt, instance)
+        except Exception as e:
+            LOG.warning('finish_revert_snapshot_based_resize_at_source failed '
+                        'during post-processing. Error: %s', e,
+                        instance=instance)
+
+    def _finish_revert_snapshot_based_resize_at_source(
+            self, ctxt, instance, migration):
+        """Private version of finish_revert_snapshot_based_resize_at_source.
+
+        This allows the main method to be decorated with error handlers.
+
+        :param ctxt: nova auth request context targeted at the source cell
+        :param instance: Instance object whose vm_state is "resized" and
+            task_state is "resize_reverting".
+        :param migration: Migration object whose status is "reverting".
+        """
+        # Delete stashed old_vm_state information. We will use this to
+        # determine if the guest should be powered on when we spawn it.
+        old_vm_state = instance.system_metadata.pop(
+            'old_vm_state', vm_states.ACTIVE)
+
+        # Update instance host/node and flavor-related fields. After this
+        # if anything fails the instance will get rebuilt/rebooted on this
+        # host.
+        self._finish_revert_resize_update_instance_flavor_host_node(
+            instance, migration)
+
+        # Move the allocations against the source compute node resource
+        # provider, held by the migration, to the instance which will drop
+        # the destination compute node resource provider allocations held by
+        # the instance. This puts the allocations against the source node
+        # back to the old_flavor and owned by the instance.
+        try:
+            self._revert_allocation(ctxt, instance, migration)
+        except exception.AllocationMoveFailed:
+            # Log the error but do not re-raise because we want to continue to
+            # process ports and volumes below.
+            LOG.error('Reverting allocation in placement for migration '
+                      '%(migration_uuid)s failed. You may need to manually '
+                      'remove the allocations for the migration consumer '
+                      'against the source node resource provider '
+                      '%(source_provider)s and the allocations for the '
+                      'instance consumer against the destination node '
+                      'resource provider %(dest_provider)s and then run the '
+                      '"nova-manage placement heal_allocations" command.',
+                      {'instance_uuid': instance.uuid,
+                       'migration_uuid': migration.uuid,
+                       'source_provider': migration.source_node,
+                       'dest_provider': migration.dest_node},
+                      instance=instance)
+
+        bdms = instance.get_bdms()
+        # prep_snapshot_based_resize_at_source created empty volume attachments
+        # that we need to update here to get the connection_info before calling
+        # driver.finish_revert_migration which will connect the volumes to this
+        # host.
+        LOG.debug('Updating volume attachments for target host %s.',
+                  self.host, instance=instance)
+        # TODO(mriedem): We should probably make _update_volume_attachments
+        # (optionally) graceful to errors so we (1) try to process all
+        # attachments and (2) continue to process networking below.
+        self._update_volume_attachments(ctxt, instance, bdms)
+
+        LOG.debug('Updating port bindings for source host %s.',
+                  self.host, instance=instance)
+        # TODO(mriedem): Calculate provider mappings when we support
+        # cross-cell resize/migrate with ports having resource requests.
+        self._finish_revert_resize_network_migrate_finish(
+            ctxt, instance, migration, provider_mappings=None)
+        network_info = self.network_api.get_instance_nw_info(ctxt, instance)
+
+        # Remember that prep_snapshot_based_resize_at_source destroyed the
+        # guest but left the disks intact so we cannot call spawn() here but
+        # finish_revert_migration should do the job.
+        block_device_info = self._get_instance_block_device_info(
+            ctxt, instance, bdms=bdms)
+        power_on = old_vm_state == vm_states.ACTIVE
+        driver_error = None
+        try:
+            self.driver.finish_revert_migration(
+                ctxt, instance, network_info, migration,
+                block_device_info=block_device_info, power_on=power_on)
+        except Exception as e:
+            driver_error = e
+            # Leave a hint about hard rebooting the guest and reraise so the
+            # instance is put into ERROR state.
+            with excutils.save_and_reraise_exception(logger=LOG):
+                LOG.error('An error occurred during finish_revert_migration. '
+                          'The instance may need to be hard rebooted. Error: '
+                          '%s', driver_error, instance=instance)
+        else:
+            # Perform final cleanup of the instance in the database.
+            instance.drop_migration_context()
+            # If the original vm_state was STOPPED, set it back to STOPPED.
+            vm_state = vm_states.ACTIVE if power_on else vm_states.STOPPED
+            self._update_instance_after_spawn(
+                ctxt, instance, vm_state=vm_state)
+            instance.save(expected_task_state=[task_states.RESIZE_REVERTING])
+        finally:
+            # Complete any volume attachments so the volumes are in-use. We
+            # do this regardless of finish_revert_migration failing because
+            # the instance is back on this host now and we do not want to leave
+            # the volumes in a pending state in case the instance is hard
+            # rebooted.
+            LOG.debug('Completing volume attachments for instance on source '
+                      'host.', instance=instance)
+            with excutils.save_and_reraise_exception(
+                    reraise=driver_error is not None, logger=LOG):
+                self._complete_volume_attachments(ctxt, bdms)
+
+        migration.status = 'reverted'
+        migration.save()
+
     @wrap_exception()
     @reverts_task_state
     @wrap_instance_event(prefix='compute')
@@ -4789,6 +4936,27 @@ def _finish_revert_resize_network_migrate_finish(
                 LOG.error('Timeout waiting for Neutron events: %s', events,
                           instance=instance)
 
+    def _finish_revert_resize_update_instance_flavor_host_node(self, instance,
+                                                               migration):
+        """Updates host/node and flavor-related fields on the instance.
+
+        This is used when finish the revert resize operation on the source
+        host and updates the instance flavor-related fields back to the old
+        flavor and then nulls out the old/new_flavor fields.
+
+        The instance host/node fields are also set back to the source compute
+        host/node.
+
+        :param instance: Instance object
+        :param migration: Migration object
+        """
+        self._set_instance_info(instance, instance.old_flavor)
+        instance.old_flavor = None
+        instance.new_flavor = None
+        instance.host = migration.source_compute
+        instance.node = migration.source_node
+        instance.save(expected_task_state=[task_states.RESIZE_REVERTING])
+
     @wrap_exception()
     @reverts_task_state
     @wrap_instance_event(prefix='compute')
@@ -4817,12 +4985,8 @@ def finish_revert_resize(
             old_vm_state = instance.system_metadata.pop('old_vm_state',
                                                         vm_states.ACTIVE)
 
-            self._set_instance_info(instance, instance.old_flavor)
-            instance.old_flavor = None
-            instance.new_flavor = None
-            instance.host = migration.source_compute
-            instance.node = migration.source_node
-            instance.save()
+            self._finish_revert_resize_update_instance_flavor_host_node(
+                instance, migration)
 
             try:
                 source_allocations = self._revert_allocation(

nova/compute/rpcapi.py

@@ -375,6 +375,7 @@ class ComputeAPI(object):
         * 5.7 - Add finish_snapshot_based_resize_at_dest()
         * 5.8 - Add confirm_snapshot_based_resize_at_source()
         * 5.9 - Add revert_snapshot_based_resize_at_dest()
+        * 5.10 - Add finish_revert_snapshot_based_resize_at_source()
     '''
 
     VERSION_ALIASES = {
@@ -732,6 +733,41 @@ def finish_snapshot_based_resize_at_dest(
             instance=instance, migration=migration, snapshot_id=snapshot_id,
             request_spec=request_spec)
 
+    def finish_revert_snapshot_based_resize_at_source(
+            self, ctxt, instance, migration):
+        """Reverts a snapshot-based resize at the source host.
+
+        Spawn the guest and re-connect volumes/VIFs on the source host and
+        revert the instance to use the old_flavor for resource usage reporting.
+
+        Updates allocations in the placement service to move the source node
+        allocations, held by the migration record, to the instance and drop
+        the allocations held by the instance on the destination node.
+
+        This is a synchronous RPC call using the ``long_rpc_timeout``
+        configuration option.
+
+        :param ctxt: nova auth request context targeted at the source cell
+        :param instance: Instance object whose vm_state is "resized" and
+            task_state is "resize_reverting".
+        :param migration: Migration object whose status is "reverting".
+        :raises: nova.exception.MigrationError if the source compute is too
+            old to perform the operation
+        :raises: oslo_messaging.exceptions.MessagingTimeout if the RPC call
+            times out
+        """
+        version = '5.10'
+        client = self.router.client(ctxt)
+        if not client.can_send_version(version):
+            raise exception.MigrationError(reason=_('Compute too old'))
+        cctxt = client.prepare(server=migration.source_compute,
+                               version=version,
+                               call_monitor_timeout=CONF.rpc_response_timeout,
+                               timeout=CONF.long_rpc_timeout)
+        return cctxt.call(
+            ctxt, 'finish_revert_snapshot_based_resize_at_source',
+            instance=instance, migration=migration)
+
     def get_console_output(self, ctxt, instance, tail_length):
         version = '5.0'
         cctxt = self.router.client(ctxt).prepare(

nova/objects/service.py

@@ -31,7 +31,7 @@
 
 
 # NOTE(danms): This is the global service version counter
-SERVICE_VERSION = 46
+SERVICE_VERSION = 47
 
 
 # NOTE(danms): This is our SERVICE_VERSION history. The idea is that any
@@ -173,6 +173,9 @@
     {'compute_rpc': '5.8'},
     # Version 46: Compute RPC v5.9: revert_snapshot_based_resize_at_dest
     {'compute_rpc': '5.9'},
+    # Version 47: Compute RPC v5.10:
+    # finish_revert_snapshot_based_resize_at_source
+    {'compute_rpc': '5.10'},
 )