Add cache_image() driver method and libvirt implementation

This adds a new virt driver method for pre-caching an image, as well
as an implementation for libvirt. This will remain unimplemented in
any driver that does not provide image caching functionality or does
not support pre-caching of images.

Related to blueprint image-precache-support

Change-Id: I52fbbcac9dc386f24ee81b3321dd0d8355e01976

Author: kk7ds

nova/tests/unit/virt/libvirt/test_driver.py

@@ -23533,6 +23533,61 @@ def test_cpu_traits_with_mode_none_on_power(self, mock_baseline, mock_cap):
 </cpu>
 '''], 1)
 
+    @mock.patch('oslo_utils.fileutils.ensure_tree')
+    @mock.patch('os.path.isdir')
+    @mock.patch('os.path.exists')
+    @mock.patch('os.utime')
+    @mock.patch('nova.virt.images.fetch_to_raw')
+    def test_cache_image_uncached(self, mock_fetch, mock_utime, mock_exists,
+                                  mock_isdir, mock_et, first_time=False):
+        # NOTE(artom): This is not actually a path on the system, since we
+        # are fully mocked out and are just testing string formatting in this
+        # test.
+        self.flags(instances_path='/nova/instances')
+        self.flags(image_cache_subdirectory_name='cache')
+        expected_fn = os.path.join('/nova/instances/cache',
+                                   imagecache.get_cache_fname('an-image'))
+
+        mock_isdir.return_value = not first_time
+        mock_exists.return_value = False
+        self.assertTrue(self.drvr.cache_image(self.context, 'an-image'))
+        mock_fetch.assert_called_once_with(self.context, 'an-image',
+                                           expected_fn)
+        mock_utime.assert_not_called()
+        mock_exists.assert_called_once_with(expected_fn)
+        mock_isdir.assert_called_once_with('/nova/instances/cache')
+        if first_time:
+            mock_et.assert_called_once_with('/nova/instances/cache')
+        else:
+            mock_et.assert_not_called()
+
+    def test_cache_image_existing_first_time(self):
+        self.test_cache_image_uncached(first_time=True)
+
+    @mock.patch('oslo_utils.fileutils.ensure_tree')
+    @mock.patch('os.path.isdir')
+    @mock.patch('os.path.exists')
+    @mock.patch('nova.privsep.path.utime')
+    @mock.patch('nova.virt.images.fetch_to_raw')
+    def test_cache_image_existing(self, mock_fetch, mock_utime, mock_exists,
+                                  mock_isdir, mock_et):
+        # NOTE(artom): This is not actually a path on the system, since we
+        # are fully mocked out and are just testing string formatting in this
+        # test.
+        self.flags(instances_path='/nova/instances')
+        self.flags(image_cache_subdirectory_name='cache')
+        expected_fn = os.path.join('/nova/instances/cache',
+                                   imagecache.get_cache_fname('an-image'))
+
+        mock_isdir.return_value = True
+        mock_exists.return_value = True
+        self.assertFalse(self.drvr.cache_image(self.context, 'an-image'))
+        mock_utime.assert_called_once_with(expected_fn)
+        mock_fetch.assert_not_called()
+        mock_exists.assert_called_once_with(expected_fn)
+        mock_et.assert_not_called()
+        mock_isdir.assert_not_called()
+
 
 class LibvirtVolumeUsageTestCase(test.NoDBTestCase):
     """Test for LibvirtDriver.get_all_volume_usage."""

nova/virt/driver.py

@@ -1582,6 +1582,22 @@ def manage_image_cache(self, context, all_instances):
         """
         pass
 
+    def cache_image(self, context, image_id):
+        """Download an image into the cache.
+
+        Used by the compute manager in response to a request to pre-cache
+        an image on the compute node. If the driver implements an image cache,
+        it should implement this method as well and perform the same action
+        as it does during an on-demand base image fetch in response to a
+        spawn.
+
+        :returns: A boolean indicating whether or not the image was fetched.
+                  True if it was fetched, or False if it already exists in
+                  the cache.
+        :raises: An Exception on error
+        """
+        raise NotImplementedError()
+
     def add_to_aggregate(self, context, aggregate, host, **kwargs):
         """Add a compute host to an aggregate.
 

nova/virt/libvirt/driver.py

@@ -9643,6 +9643,41 @@ def _cleanup_remote_migration(self, dest, inst_base, inst_base_resize,
         except Exception:
             pass
 
+    def cache_image(self, context, image_id):
+        cache_dir = os.path.join(CONF.instances_path,
+                                 CONF.image_cache_subdirectory_name)
+        path = os.path.join(cache_dir,
+                            imagecache.get_cache_fname(image_id))
+        if os.path.exists(path):
+            LOG.info('Image %(image_id)s already cached; updating timestamp',
+                     {'image_id': image_id})
+            # NOTE(danms): The regular image cache routines use a wrapper
+            # (_update_utime_ignore_eacces()) around this to avoid failing
+            # on permissions (which may or may not be legit due to an NFS
+            # race). However, since this is best-effort, errors are swallowed
+            # by compute manager per-image, and we are compelled to report
+            # errors up our stack, we use the raw method here to avoid the
+            # silent ignore of the EACCESS.
+            nova.privsep.path.utime(path)
+            return False
+        else:
+            # NOTE(danms): In case we are running before the first boot, make
+            # sure the cache directory is created
+            if not os.path.isdir(cache_dir):
+                fileutils.ensure_tree(cache_dir)
+            LOG.info('Caching image %(image_id)s by request',
+                     {'image_id': image_id})
+            # NOTE(danms): The imagebackend code, as called via spawn() where
+            # images are normally cached, uses a lock on the root disk it is
+            # creating at the time, but relies on the
+            # compute_utils.disk_ops_semaphore for cache fetch mutual
+            # exclusion, which is grabbed in images.fetch() (which is called
+            # by images.fetch_to_raw() below). So, by calling fetch_to_raw(),
+            # we are sharing the same locking for the cache fetch as the
+            # rest of the code currently called only from spawn().
+            images.fetch_to_raw(context, image_id, path)
+            return True
+
     def _is_storage_shared_with(self, dest, inst_base):
         # NOTE (rmk): There are two methods of determining whether we are
         #             on the same filesystem: the source and dest IP are the