Add support for geojson labels (#109)

Add support for geojson labels

Author: drewbo

.circleci/config.yml

@@ -15,6 +15,11 @@ jobs:
             sudo pip install numpy==1.13.3
             sudo pip install -r requirements.txt
             sudo pip install -r requirements-dev.txt
+            git clone [email protected]:mapbox/tippecanoe.git
+            cd tippecanoe
+            sudo make -j
+            sudo make install
+
       - run:
           name: Pylint
           command: pylint ~/label-maker/label_maker --rcfile=~/label-maker/.config/pylintrc

docs/parameters.rst

@@ -8,6 +8,9 @@ Here is the full list of configuration parameters you can specify in a ``config.
 **bounding_box**: list of floats
 	The bounding box to create images from. This should be given in the form: ``[xmin, ymin, xmax, ymax]`` as longitude and latitude values between ``[-180, 180]`` and ``[-90, 90]``, respectively. Values should use the WGS84 datum, with longitude and latitude units in decimal degrees.
 
+**geojson**: string
+	An input ``GeoJSON`` file containing labels. Adding this parameter will override the values in the ``country`` and ``bounding_box`` parameters. The ``GeoJSON`` should only contain `Polygon` and not `Multipolygon` or a `GeometryCollection`.
+
 **zoom**: int
 	The `zoom level <http://wiki.openstreetmap.org/wiki/Zoom_levels>`_ used to create images. This functions as a rough proxy for resolution. Value should be given as an int on the interval [0, 19].
 

label_maker/label.py

@@ -58,6 +58,8 @@ def make_labels(dest_folder, zoom, country, classes, ml_type, bounding_box, spar
         as longitude and latitude values between `[-180, 180]` and `[-90, 90]` respectively
     sparse: boolean
         Limit the total background tiles to write based on `background_ratio` kwarg.
+    geojson: str
+        Filepath to optional geojson label input
     **kwargs: dict
         Other properties from CLI config passed as keywords to other utility functions
     """
@@ -66,15 +68,18 @@ def make_labels(dest_folder, zoom, country, classes, ml_type, bounding_box, spar
     mbtiles_file_zoomed = op.join(dest_folder, '{}-z{!s}.mbtiles'.format(country, zoom))
 
     if not op.exists(mbtiles_file_zoomed):
-        print('Retiling QA Tiles to zoom level {} (takes a bit)'.format(zoom))
-        filtered_geo = op.join(dest_folder, '{}.geojson'.format(country))
-        ps = Popen(['tippecanoe-decode', '-c', '-f', mbtiles_file], stdout=PIPE)
-        stream_filter_fpath = op.join(op.dirname(label_maker.__file__), 'stream_filter.py')
-        run(['python', stream_filter_fpath, json.dumps(bounding_box)],
-            stdin=ps.stdout, stdout=open(filtered_geo, 'w'))
-        ps.wait()
-        run(['tippecanoe', '--no-feature-limit', '--no-tile-size-limit', '-P',
-             '-l', 'osm', '-f', '-z', str(zoom), '-Z', str(zoom), '-o',
+        filtered_geo = kwargs.get('geojson') or op.join(dest_folder, '{}.geojson'.format(country))
+        fast_parse = []
+        if not op.exists(filtered_geo):
+            fast_parse = ['-P']
+            print('Retiling QA Tiles to zoom level {} (takes a bit)'.format(zoom))
+            ps = Popen(['tippecanoe-decode', '-c', '-f', mbtiles_file], stdout=PIPE)
+            stream_filter_fpath = op.join(op.dirname(label_maker.__file__), 'stream_filter.py')
+            run(['python', stream_filter_fpath, json.dumps(bounding_box)],
+                stdin=ps.stdout, stdout=open(filtered_geo, 'w'))
+            ps.wait()
+        run(['tippecanoe', '--no-feature-limit', '--no-tile-size-limit'] + fast_parse +
+            ['-l', 'osm', '-f', '-z', str(zoom), '-Z', str(zoom), '-o',
              mbtiles_file_zoomed, filtered_geo])
 
     # Call tilereduce

label_maker/main.py

@@ -7,6 +7,7 @@
 from os import makedirs, path as op
 
 from cerberus import Validator
+from shapely.geometry import MultiPolygon, Polygon
 
 from label_maker.version import __version__
 from label_maker.download import download_mbtiles
@@ -18,6 +19,11 @@
 
 logger = logging.getLogger(__name__)
 
+def get_bounds(feature_collection):
+    """Get a bounding box for a FeatureCollection of Polygon Features"""
+    features = [f for f in feature_collection['features'] if f['geometry']['type'] in ['Polygon']]
+    return MultiPolygon(list(map(lambda x: Polygon(x['geometry']['coordinates'][0]), features))).bounds
+
 
 def parse_args(args):
     """Create an argument parser with subcommands"""
@@ -77,6 +83,16 @@ def cli():
     if not valid:
         raise Exception(v.errors)
 
+    # custom validation for top level keys
+    # require either: country & bounding_box or geojson
+    if 'geojson' not in config.keys() and not ('country' in config.keys() and 'bounding_box' in config.keys()):
+        raise Exception('either "geojson" or "country" and "bounding_box" must be present in the configuration JSON')
+
+    # for geojson, overwrite other config keys to correct labeling
+    if 'geojson' in config.keys():
+        config['country'] = op.splitext(op.basename(config.get('geojson')))[0]
+        config['bounding_box'] = get_bounds(json.load(open(config.get('geojson'), 'r')))
+
     if cmd == 'download':
         download_mbtiles(dest_folder=dest_folder, **config)
     elif cmd == 'labels':

label_maker/validate.py

@@ -21,8 +21,9 @@
 lon_schema = {'type': 'float', 'min': -180, 'max': 180}
 
 schema = {
-    'country': {'type': 'string', 'allowed': countries, 'required': True},
-    'bounding_box': {'type': 'list', 'items': [lon_schema, lat_schema, lon_schema, lat_schema], 'required': True},
+    'geojson': {'type': 'string'},
+    'country': {'type': 'string', 'allowed': countries},
+    'bounding_box': {'type': 'list', 'items': [lon_schema, lat_schema, lon_schema, lat_schema]},
     'zoom': {'type': 'integer', 'required': True},
     'classes': {'type': 'list', 'schema': class_schema, 'required': True},
     'imagery': {'type': 'string', 'required': True},

test/fixtures/integration/config.geojson.json

@@ -0,0 +1,16 @@
+{
+  "geojson": "integration-cl/labels.geojson",
+  "zoom": 17,
+  "classes": [
+    { "name": "Water Tower", "filter": ["==", "man_made", "water_tower"] },
+    { "name": "Building", "filter": ["has", "building"] },
+    { "name": "Farmland", "filter": ["==", "landuse", "farmland"] },
+    { "name": "Ruins", "filter": ["==", "historic", "ruins"] },
+    { "name": "Parking", "filter": ["==", "amenity", "parking"] },
+    { "name": "Roads", "filter": ["has", "highway"] }
+  ],
+  "imagery": "https://api.mapbox.com/v4/mapbox.satellite/{z}/{x}/{y}.jpg?access_token=ACCESS_TOKEN",
+  "background_ratio": 1,
+  "ml_type": "classification",
+  "seed": 19
+}

test/fixtures/integration/labels.geojson

@@ -0,0 +1,10 @@
+{
+  "geojson": "test.geojson",
+  "zoom": 16,
+  "classes": [
+    { "name": "Substation", "filter": ["==", "power", "substation"] }
+  ],
+  "imagery": "http://a.tiles.mapbox.com/v4/mapbox.satellite/{z}/{x}/{y}.jpg",
+  "background_ratio": 1,
+  "ml_type": "classification"
+}

test/fixtures/validation/geojson.json

@@ -63,5 +63,4 @@ def test_cli(self):
                 expected_geojson = json.load(fixture)
                 geojson = json.load(geojson_file)
 
-                for feature in geojson['features']:
-                    self.assertTrue(feature in expected_geojson['features'])
+                self.assertCountEqual(expected_geojson, geojson)

test/integration/test_classification_labels.py

@@ -0,0 +1,66 @@
+"""Test that the following CLI command returns the expected outputs
+label-maker labels --dest integration-cl --config test/fixtures/integration/config.geojson.json"""
+import unittest
+import json
+from os import makedirs
+from shutil import copyfile, rmtree
+import subprocess
+
+import numpy as np
+
+class TestClassificationLabelGeoJSON(unittest.TestCase):
+    """Tests for classification label creation"""
+    @classmethod
+    def setUpClass(cls):
+        makedirs('integration-cl')
+        copyfile('test/fixtures/integration/labels.geojson', 'integration-cl/labels.geojson')
+
+    @classmethod
+    def tearDownClass(cls):
+        rmtree('integration-cl')
+
+    def test_cli(self):
+        """Verify stdout, geojson, and labels.npz produced by CLI"""
+        # our command line output should look like this
+        expected_output = """Determining labels for each tile
+---
+Water Tower: 1 tiles
+Building: 1 tiles
+Farmland: 0 tiles
+Ruins: 1 tiles
+Parking: 1 tiles
+Roads: 8 tiles
+Total tiles: 9
+Writing out labels to integration-cl/labels.npz
+"""
+
+        cmd = 'label-maker labels --dest integration-cl --config test/fixtures/integration/config.geojson.json'
+        cmd = cmd.split(' ')
+        with subprocess.Popen(cmd, universal_newlines=True, stdout=subprocess.PIPE) as p:
+            self.assertEqual(expected_output, p.stdout.read())
+
+        # our labels should look like this
+        expected_labels = {
+            '62092-50162-17': np.array([1, 0, 0, 0, 0, 0, 0]),
+            '62092-50163-17': np.array([0, 0, 0, 0, 0, 0, 1]),
+            '62092-50164-17': np.array([0, 0, 0, 0, 0, 0, 1]),
+            '62093-50162-17': np.array([0, 0, 0, 0, 0, 0, 1]),
+            '62093-50164-17': np.array([0, 0, 0, 0, 0, 0, 1]),
+            '62094-50162-17': np.array([0, 0, 0, 0, 0, 0, 1]),
+            '62094-50164-17': np.array([0, 0, 0, 0, 0, 0, 1]),
+            '62094-50163-17': np.array([0, 1, 1, 0, 0, 0, 1]),
+            '62093-50163-17': np.array([0, 0, 0, 0, 1, 1, 1])
+        }
+
+        labels = np.load('integration-cl/labels.npz')
+        self.assertEqual(len(labels.files), len(expected_labels.keys()))  # First check number of tiles
+        for tile in labels.files:
+            self.assertTrue(np.array_equal(expected_labels[tile], labels[tile]))  # Now, content
+
+        # our GeoJSON looks like the fixture
+        with open('test/fixtures/integration/classification.geojson') as fixture:
+            with open('integration-cl/classification.geojson') as geojson_file:
+                expected_geojson = json.load(fixture)
+                geojson = json.load(geojson_file)
+
+                self.assertCountEqual(expected_geojson, geojson)

test/integration/test_classification_labels_geojson.py

@@ -63,5 +63,4 @@ def test_cli(self):
                 expected_geojson = json.load(fixture)
                 geojson = json.load(geojson_file)
 
-                for feature in geojson['features']:
-                    self.assertTrue(feature in expected_geojson['features'])
+                self.assertCountEqual(expected_geojson, geojson)

test/integration/test_classification_labels_sparse.py

@@ -58,5 +58,4 @@ def test_cli(self):
                 expected_geojson = json.load(fixture)
                 geojson = json.load(geojson_file)
 
-                for feature in geojson['features']:
-                    self.assertTrue(feature in expected_geojson['features'])
+                self.assertCountEqual(expected_geojson, geojson)

test/integration/test_directory_move.py

@@ -21,12 +21,19 @@ def test_required(self):
         """Test for all required keys"""
         with open('test/fixtures/validation/passing.json') as config_file:
             config = json.load(config_file)
-            for key in ['country', 'bounding_box', 'zoom', 'classes', 'imagery', 'ml_type']:
+            for key in ['zoom', 'classes', 'imagery', 'ml_type']:
                 bad_config = copy.deepcopy(config)
                 bad_config.pop(key)
                 valid = v.validate(bad_config)
                 self.assertFalse(valid)
 
+    def test_geojson(self):
+        """Test an alternate configuration with geojson input"""
+        with open('test/fixtures/validation/geojson.json') as config_file:
+            config = json.load(config_file)
+            valid = v.validate(config)
+            self.assertTrue(valid)
+
     def test_country(self):
         """Test country not in list fails"""
         with open('test/fixtures/validation/passing.json') as config_file: