Merge pull request #94 from developmentseed/feature/93-wms

Feature/93 wms

Author: wronk

README.md

@@ -47,6 +47,8 @@ Before running any commands, it is necessary to create a `config.json` file to s
 - `imagery`: One of:
   - A template string for a tiled imagery service. Note that you will generally need an API key to obtain images and there may be associated costs. The above example requires a [Mapbox access token](https://www.mapbox.com/help/how-access-tokens-work/)
   - A GeoTIFF file location. Works with both local and remote files. Ex: `'http://oin-hotosm.s3.amazonaws.com/593ede5ee407d70011386139/0/3041615b-2bdb-40c5-b834-36f580baca29.tif'`
+  - A [WMS endpoint](http://www.opengeospatial.org/standards/wms) `GetMap` request. Fill out all necessary parameters except `bbox` which should be set as `{bbox}`. Ex:
+  `'https://basemap.nationalmap.gov/arcgis/services/USGSImageryOnly/MapServer/WMSServer?SERVICE=WMS&REQUEST=GetMap&VERSION=1.1.1&LAYERS=0&STYLES=&FORMAT=image%2Fjpeg&TRANSPARENT=false&HEIGHT=256&WIDTH=256&SRS=EPSG%3A3857&BBOX={bbox}'`
 - `background_ratio`: For single-class classification problems, we need to download images with no matching class. We will download `background_ratio` times the number of images matching the one class.
 - `ml_type`: One of `"classification"`, `"object-detection"`, or `"segmentation"`. For the final label numpy arrays (`y_train` and `y_test`), we will produce a different label depending upon the `type`.
   - `"classification"`: An array of the same length as `classes`. Each array value will be either `1` or `0` based on whether it matches the class at the same index

label_maker/countries.txt

@@ -197,7 +197,7 @@ south_georgia_and_south_sandwich_islands
 south_korea_korea
 south_sudan
 spain
-sri_lanka_lanka
+sri_lanka
 sudan
 suriname
 swaziland

label_maker/images.py

@@ -6,7 +6,7 @@
 
 import numpy as np
 
-from label_maker.utils import download_tile_tms, get_tile_tif, is_tif
+from label_maker.utils import get_image_function
 
 def download_images(dest_folder, classes, imagery, ml_type, background_ratio, imagery_offset=False, **kwargs):
     """Download satellite images specified by a URL and a label.npz file
@@ -69,9 +69,7 @@ def class_test(value):
     print('Downloading {} tiles to {}'.format(len(tiles), tiles_dir))
 
     # get image acquisition function based on imagery string
-    image_function = download_tile_tms
-    if is_tif(imagery):
-        image_function = get_tile_tif
+    image_function = get_image_function(imagery)
 
     for tile in tiles:
         image_function(tile, imagery, tiles_dir, imagery_offset)

label_maker/preview.py

@@ -7,7 +7,7 @@
 import numpy as np
 from PIL import Image, ImageDraw
 
-from label_maker.utils import class_match, download_tile_tms, get_tile_tif, is_tif
+from label_maker.utils import class_match, get_image_function
 
 def preview(dest_folder, number, classes, imagery, ml_type, imagery_offset=False, **kwargs):
     """Produce imagery examples for specified classes
@@ -48,9 +48,7 @@ def preview(dest_folder, number, classes, imagery, ml_type, imagery_offset=False
     print('Writing example images to {}'.format(examples_dir))
 
     # get image acquisition function based on imagery string
-    image_function = download_tile_tms
-    if is_tif(imagery):
-        image_function = get_tile_tif
+    image_function = get_image_function(imagery)
 
     for i, cl in enumerate(classes):
         # create class directory

label_maker/utils.py

@@ -1,7 +1,7 @@
 # pylint: disable=unused-argument
 """Provide utility functions"""
 from os import path as op
-from urllib.parse import urlparse
+from urllib.parse import urlparse, parse_qs
 
 from mercantile import bounds
 from pyproj import Proj, transform
@@ -30,7 +30,8 @@ def download_tile_tms(tile, imagery, folder, *args):
     _, image_format = op.splitext(o.path)
     r = requests.get(url(tile.split('-'), imagery))
     tile_img = op.join(folder, '{}{}'.format(tile, image_format))
-    open(tile_img, 'wb').write(r.content)
+    with open(tile_img, 'wb')as w:
+        w.write(r.content)
     return tile_img
 
 def get_tile_tif(tile, imagery, folder, imagery_offset):
@@ -86,6 +87,49 @@ def get_tile_tif(tile, imagery, folder, imagery_offset):
 
     return tile_img
 
+def get_tile_wms(tile, imagery, folder, imagery_offset):
+    """
+    Read a WMS endpoint with query parameters corresponding to a TMS tile
+
+    Converts the tile boundaries to the spatial reference system (SRS) specified
+    by the WMS query parameter.
+    """
+    # retrieve the necessary parameters from the query string
+    query_dict = parse_qs(imagery.lower())
+    image_format = query_dict['format'][0].split('/')[1]
+    wms_srs = query_dict['srs'][0]
+
+    # find our tile bounding box
+    bound = bounds(*[int(t) for t in tile.split('-')])
+    p1 = Proj({'init': 'epsg:4326'})
+    p2 = Proj({'init': wms_srs})
+
+    # project the tile bounding box from lat/lng to WMS SRS
+    tile_ll_proj = transform(p1, p2, bound.west, bound.south)
+    tile_ur_proj = transform(p1, p2, bound.east, bound.north)
+    bbox = tile_ll_proj + tile_ur_proj
+
+    # request the image with the transformed bounding box and save
+    wms_url = imagery.replace('{bbox}', ','.join([str(b) for b in bbox]))
+    r = requests.get(wms_url)
+    tile_img = op.join(folder, '{}.{}'.format(tile, image_format))
+    with open(tile_img, 'wb') as w:
+        w.write(r.content)
+    return tile_img
+
+
 def is_tif(imagery):
     """Determine if an imagery path has a valid tif extension"""
     return op.splitext(imagery)[1].lower() in ['.tif', '.tiff', '.vrt']
+
+def is_wms(imagery):
+    """Determine if an imagery path is a WMS endpoint"""
+    return '{bbox}' in imagery
+
+def get_image_function(imagery):
+    """Return the correct image downloading function based on the imagery string"""
+    if is_tif(imagery):
+        return get_tile_tif
+    if is_wms(imagery):
+        return get_tile_wms
+    return download_tile_tms

requirements-dev.txt

@@ -1,3 +1,3 @@
 astroid>=1.6.0
 isort>=4.2.15
-pylint>=1.8.1
+pylint==1.8.1

test/fixtures/4686-6267-14.jpeg

@@ -5,7 +5,7 @@
 import numpy as np
 from PIL import Image
 
-from label_maker.utils import url, class_match, get_tile_tif
+from label_maker.utils import url, class_match, get_tile_tif, get_tile_wms
 
 class TestUtils(unittest.TestCase):
     """Tests for utility functions"""
@@ -85,5 +85,21 @@ def test_get_tile_vrt(self):
         fixture_tile = Image.open('test/fixtures/{}.jpg'.format(tile))
         self.assertEqual(test_tile, fixture_tile)
 
+    def test_get_tile_wms(self):
+        """Test reading of tile from a WMS endpoint"""
+        tile = '4686-6267-14'
+        # create tiles directory
+        dest_folder = 'test'
+        tiles_dir = op.join(dest_folder, 'tiles')
+        if not op.isdir(tiles_dir):
+            makedirs(tiles_dir)
+
+        usgs_url = 'https://basemap.nationalmap.gov/arcgis/services/USGSImageryOnly/MapServer/WMSServer?SERVICE=WMS&REQUEST=GetMap&VERSION=1.1.1&LAYERS=0&STYLES=&FORMAT=image%2Fjpeg&TRANSPARENT=false&HEIGHT=256&WIDTH=256&SRS=EPSG%3A3857&BBOX={bbox}'
+
+        get_tile_wms(tile, usgs_url, tiles_dir, None)
+        test_tile = Image.open('test/tiles/{}.jpeg'.format(tile))
+        fixture_tile = Image.open('test/fixtures/{}.jpeg'.format(tile))
+        self.assertEqual(test_tile, fixture_tile)
+
 if __name__ == '__main__':
     unittest.main()