Basic structure for comparing generated plots in unit tests

Co-authored-by: Tim Treis <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: 3 people

.github/workflows/test.yaml

@@ -55,6 +55,18 @@ jobs:
                   DISPLAY: :42
               run: |
                   pytest -v --cov --color=yes --cov-report=xml
+            # - name: Generate GH action "groundtruth" figures as artifacts, uncomment if needed
+            #   if: always()
+            #   uses: actions/upload-artifact@v3
+            #   with:
+            #       name: groundtruth-figures
+            #       path: /home/runner/work/spatialdata-plot/spatialdata-plot/tests/_images/*
+            - name: Archive figures generated during testing
+              if: always()
+              uses: actions/upload-artifact@v3
+              with:
+                  name: plotting-results
+                  path: /home/runner/work/spatialdata-plot/spatialdata-plot/tests/figures/*
             - name: Upload coverage to Codecov
               uses: codecov/[email protected]
               with:

.gitignore

@@ -29,5 +29,9 @@ __pycache__/
 
 format.sh
 
+
+# test
+tests/figures/
+
 # jupyter checkpoints
 .ipynb_checkpoints

tests/__init__.py

@@ -1,17 +1,23 @@
-from typing import Optional, Union
+from abc import ABC, ABCMeta
+from functools import wraps
+from pathlib import Path
+from typing import Callable, Optional, Union
 
+import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
 import pyarrow as pa
 import pytest
 import spatialdata as sd
 from anndata import AnnData
 from geopandas import GeoDataFrame
+from matplotlib.testing.compare import compare_images
 from multiscale_spatial_image import MultiscaleSpatialImage
 from numpy.random import default_rng
 from shapely.geometry import MultiPolygon, Polygon
 from spatial_image import SpatialImage
 from spatialdata import SpatialData
+from spatialdata.datasets import blobs
 from spatialdata.models import (
     Image2DModel,
     Image3DModel,
@@ -25,6 +31,13 @@
 
 import spatialdata_plot  # noqa: F401
 
+HERE: Path = Path(__file__).parent
+
+EXPECTED = HERE / "_images"
+ACTUAL = HERE / "figures"
+TOL = 60
+DPI = 40
+
 RNG = default_rng()
 
 
@@ -39,6 +52,11 @@ def full_sdata() -> SpatialData:
     )
 
 
+@pytest.fixture()
+def sdata_blobs() -> SpatialData:
+    return blobs()
+
+
 @pytest.fixture
 def test_sdata_single_image():
     """Creates a simple sdata object."""
@@ -152,17 +170,6 @@ def table_multiple_annotations() -> SpatialData:
     return SpatialData(table=_get_table(region=["sample1", "sample2"]))
 
 
-# @pytest.fixture()
-# def empty_points() -> SpatialData:
-#     geo_df = GeoDataFrame(
-#         geometry=[],
-#     )
-#     from spatialdata.transformations import Identity
-#     set_transform(geo_df, Identity())
-#
-#     return SpatialData(points={"empty": geo_df})
-
-
 @pytest.fixture()
 def empty_table() -> SpatialData:
     adata = AnnData(shape=(0, 0))
@@ -337,3 +344,48 @@ def _get_table(
         return TableModel.parse(adata=adata, region=region, region_key=region_key, instance_key=instance_key)
     else:
         return TableModel.parse(adata=adata, region=region, region_key=region_key, instance_key=instance_key)
+
+
+class PlotTesterMeta(ABCMeta):
+    def __new__(cls, clsname, superclasses, attributedict):
+        for key, value in attributedict.items():
+            if callable(value):
+                attributedict[key] = _decorate(value, clsname, name=key)
+        return super().__new__(cls, clsname, superclasses, attributedict)
+
+
+class PlotTester(ABC):  # noqa: B024
+    @classmethod
+    def compare(cls, basename: str, tolerance: Optional[float] = None):
+        ACTUAL.mkdir(parents=True, exist_ok=True)
+        out_path = ACTUAL / f"{basename}.png"
+
+        plt.savefig(out_path, dpi=DPI)
+        plt.close()
+
+        if tolerance is None:
+            # see https://github.com/scverse/squidpy/pull/302
+            tolerance = 2 * TOL if "Napari" in str(basename) else TOL
+
+        res = compare_images(str(EXPECTED / f"{basename}.png"), str(out_path), tolerance)
+
+        assert res is None, res
+
+
+def _decorate(fn: Callable, clsname: str, name: Optional[str] = None) -> Callable:
+    @wraps(fn)
+    def save_and_compare(self, *args, **kwargs):
+        fn(self, *args, **kwargs)
+        self.compare(fig_name)
+
+    if not callable(fn):
+        raise TypeError(f"Expected a `callable` for class `{clsname}`, found `{type(fn).__name__}`.")
+
+    name = fn.__name__ if name is None else name
+
+    if not name.startswith("test_plot_") or not clsname.startswith("Test"):
+        return fn
+
+    fig_name = f"{clsname[4:]}_{name[10:]}"
+
+    return save_and_compare

tests/_images/Images_images.png

@@ -0,0 +1,27 @@
+import matplotlib
+import scanpy as sc
+from spatialdata import SpatialData
+
+import spatialdata_plot  # noqa: F401
+from tests.conftest import PlotTester, PlotTesterMeta
+
+sc.pl.set_rcParams_defaults()
+sc.set_figure_params(dpi=40, color_map="viridis")
+matplotlib.use("agg")  # same as GitHub action runner
+
+# WARNING:
+# 1. all classes must both subclass PlotTester and use metaclass=PlotTesterMeta
+# 2. tests which produce a plot must be prefixed with `test_plot_`
+# 3. if the tolerance needs to be changed, don't prefix the function with `test_plot_`, but with something else
+#    the comp. function can be accessed as `self.compare(<your_filename>, tolerance=<your_tolerance>)`
+#    ".png" is appended to <your_filename>, no need to set it
+
+
+class TestLabels(PlotTester, metaclass=PlotTesterMeta):
+    def test_plot_labels(self, sdata_blobs: SpatialData):
+        sdata_blobs.pl.render_labels(color="channel_2_mean").pl.show()
+
+
+class TestImages(PlotTester, metaclass=PlotTesterMeta):
+    def test_plot_images(self, sdata_blobs: SpatialData):
+        sdata_blobs.pl.render_images().pl.show()