fix: CachedRegionsProvider shrink not working

Author: lihsai0

qiniu/http/regions_provider.py

@@ -319,7 +319,7 @@ def _parse_persisted_regions(persisted_data):
         _get_region_from_persisted(d)
         for d in parsed_data.get('regions', [])
     ]
-    return parsed_data.get('cache_key'), regions
+    return parsed_data.get('cacheKey'), regions
 
 
 def _walk_persist_cache_file(persist_path, ignore_parse_error=False):
@@ -601,15 +601,17 @@ def __should_shrink(self):
         -------
         bool
         """
-        return self._cache_scope.last_shrink_at + self._cache_scope.shrink_interval >= datetime.datetime.now()
+        return datetime.datetime.now() - self._cache_scope.last_shrink_at > self._cache_scope.shrink_interval
 
     def __shrink_cache(self):
         # shrink memory cache
         if self._cache_scope.should_shrink_expired_regions:
+            kept_memo_cache = {}
             for k, regions in self._cache_scope.memo_cache.items():
-                live_regions = [r.is_live for r in regions]
+                live_regions = [r for r in regions if r.is_live]
                 if live_regions:
-                    self._cache_scope.memo_cache[k] = live_regions
+                    kept_memo_cache[k] = live_regions
+            self._cache_scope = self._cache_scope._replace(memo_cache=kept_memo_cache)
 
         # shrink file cache
         if not self._cache_scope.persist_path:

tests/cases/test_http/test_regions_provider.py

@@ -95,6 +95,18 @@ def cached_regions_provider(request):
             pass
 
 
+@pytest.fixture(scope='function')
+def bad_regions_provider():
+    regions_provider = QueryRegionsProvider(
+        access_key='fake',
+        bucket_name='fake',
+        endpoints_provider=[
+            Endpoint('fake-uc.python.qiniu.com')
+        ]
+    )
+    yield regions_provider
+
+
 class TestCachedQueryRegionsProvider:
     @pytest.mark.parametrize(
         'cached_regions_provider',
@@ -178,14 +190,78 @@ def test_getter_with_base_regions_provider(self, cached_regions_provider):
                 line_num += 1
         assert line_num == 1
 
-    def test_should_provide_memo_expired_regions_when_base_provider_failed(self):
-        pass
-
-    def test_should_provide_file_expired_regions_when_base_provider_failed(self):
-        pass
+    @pytest.mark.parametrize(
+        'cached_regions_provider',
+        [
+            {
+                'persist_path': os.path.join(tempfile.gettempdir(), 'test-base-provider.jsonl')
+            }
+        ],
+        indirect=True
+    )
+    def test_should_provide_memo_expired_regions_when_base_provider_failed(
+        self,
+        cached_regions_provider,
+        bad_regions_provider
+    ):
+        expired_region = Region.from_region_id('z0')
+        expired_region.create_time = datetime.datetime.fromtimestamp(0)
+        expired_region.ttl = 1
+        cached_regions_provider.set_regions([expired_region])
+        cached_regions_provider.base_regions_provider = bad_regions_provider
+        regions = list(cached_regions_provider)
+        assert len(regions) > 0
+        assert not regions[0].is_live
 
-    def test_shrink_normally(self):
-        pass
+    @pytest.mark.parametrize(
+        'cached_regions_provider',
+        [
+            {
+                'persist_path': os.path.join(tempfile.gettempdir(), 'test-base-provider.jsonl')
+            }
+        ],
+        indirect=True
+    )
+    def test_should_provide_file_expired_regions_when_base_provider_failed(
+        self,
+        cached_regions_provider,
+        bad_regions_provider
+    ):
+        expired_region = Region.from_region_id('z0')
+        expired_region.create_time = datetime.datetime.fromtimestamp(0)
+        expired_region.ttl = 1
+        cached_regions_provider.set_regions([expired_region])
+        cached_regions_provider._cache_scope.memo_cache.clear()
+        cached_regions_provider.base_regions_provider = bad_regions_provider
+        regions = list(cached_regions_provider)
+        assert len(regions) > 0
+        assert not regions[0].is_live
 
-    def test_shrink_with_ignore_expired_regions(self):
-        pass
+    @pytest.mark.parametrize(
+        'cached_regions_provider',
+        [
+            {
+                'should_shrink_expired_regions': True
+            }
+        ],
+        indirect=True
+    )
+    def test_shrink_with_expired_regions(self, cached_regions_provider):
+        expired_region = Region.from_region_id('z0')
+        expired_region.create_time = datetime.datetime.fromtimestamp(0)
+        expired_region.ttl = 1
+        origin_cache_key = cached_regions_provider.cache_key
+        cached_regions_provider.set_regions([expired_region])
+        cached_regions_provider.cache_key = 'another-cache-key'
+        list(cached_regions_provider)  # trigger __shrink_cache()
+        assert len(cached_regions_provider._cache_scope.memo_cache[origin_cache_key]) == 0
+
+    def test_shrink_with_ignore_expired_regions(self, cached_regions_provider):
+        expired_region = Region.from_region_id('z0')
+        expired_region.create_time = datetime.datetime.fromtimestamp(0)
+        expired_region.ttl = 1
+        origin_cache_key = cached_regions_provider.cache_key
+        cached_regions_provider.set_regions([expired_region])
+        cached_regions_provider.cache_key = 'another-cache-key'
+        list(cached_regions_provider)  # trigger __shrink_cache()
+        assert len(cached_regions_provider._cache_scope.memo_cache[origin_cache_key]) > 0