40 potential bug tb in geofabrics test suite related to geopais lidar (…

…#41) * Updated test to reflect improved open topography checks for dataset boundaries * Remove shapely warning * Updated to use logging * Updated package version * Minor print out tidy up * fixup: Format Python code with Black --------- Co-authored-by: github-actions <[email protected]>
niwa · Jun 6, 2024 · 17db805 · 17db805
1 parent 048796b
commit 17db805
Show file tree

Hide file tree

Showing 12 changed files with 44 additions and 53 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "geoapis"
-version = "0.3.2"
+version = "0.3.3"
 description = "A package for downloading geospatial data from web APIs."
 readme = "README.md"
 authors = [{ name = "Rose pearson", email = "[email protected]" }]
@@ -27,7 +27,7 @@ dependencies = [
     "boto3",
     "python-dotenv",
     "tqdm",
-    'tomli; python_version < "3.11"',
+    'tomli; python_version < "3.6"',
 ]
 requires-python = ">=3.6"
 

diff --git a/src/geoapis/__init__.py b/src/geoapis/__init__.py
@@ -4,4 +4,4 @@
 
 @author: pearsonra
 """
-__version__ = "0.3.2"
+__version__ = "0.3.3"
diff --git a/src/geoapis/lidar.py b/src/geoapis/lidar.py
@@ -7,6 +7,7 @@
 
 import urllib
 import pathlib
+import logging
 import requests
 import boto3
 import botocore
@@ -81,11 +82,11 @@ def run(self, dataset_name: str = None):
         elif self.search_polygon is not None:
             self.download_datasets_in_polygon()
         else:
-            print(
+            logging.info(
                 "Both the search_polygon and dataset_name are None. Either a "
-                "dataset_name of search polygon needs to be specified if any datasets "
+                "'dataset_name' or 'search_polygon' needs to be specified if any datasets "
                 "are to be downloaded from OpenTopography. Please specify a "
-                "search_polygon during construction, or a dataset_name during run."
+                "'search_polygon' during construction, or a 'dataset_name' during run."
             )
 
     def download_datasets_in_polygon(self):
@@ -167,7 +168,7 @@ def download_dataset(self, dataset_prefix, client):
         dataset_prefix"""
 
         if self.verbose:
-            print(f"Check files in dataset {dataset_prefix}")
+            logging.info(f"Check files in dataset {dataset_prefix}")
         tile_info = self._get_dataset_tile_names(client, dataset_prefix)
 
         # check download size limit is not exceeded
@@ -196,7 +197,6 @@ def _get_dataset_tile_names(self, client, dataset_prefix):
 
         # Download the file if needed
         if self.redownload_files_bool or not local_file_path.exists():
-
             # ensure folder exists before download
             local_file_path.parent.mkdir(parents=True, exist_ok=True)
             try:
@@ -219,15 +219,14 @@ def _calculate_dataset_download_size(self, client, _dataset_prefix, tile_info):
             )
             local_path = self.cache_path / file_name
             if self.redownload_files_bool or not local_path.exists():
-
                 try:
                     response = client.head_object(
                         Bucket=self.OT_BUCKET, Key=str(file_name.as_posix())
                     )
                     lidar_size_bytes += response["ContentLength"]
 
                     if self.verbose:
-                        print(
+                        logging.info(
                             f"checking size: {file_name}: {response['ContentLength']}"
                             f", total (GB): {self._to_gbytes(lidar_size_bytes)}"
                         )

diff --git a/src/geoapis/vector.py b/src/geoapis/vector.py
@@ -6,6 +6,7 @@
 """
 
 import urllib
+import logging
 import requests
 import shapely
 import shapely.geometry
@@ -136,12 +137,10 @@ def get_json_response_in_bounds(self, layer: int, bounds, geometry_name: str):
         # If a geometry_name was specified use this, otherwise try the standard LINZ
         # ones
         if geometry_name is not None and geometry_name != "":
-
             response = self.query_vector_wfs_in_bounds(layer, bounds, geometry_name)
             response.raise_for_status()
             return response.json()
         else:
-
             # cycle through the standard geometry_name's - suppress errors and only
             # raise one if no valid responses
             for geometry_name in self.GEOMETRY_NAMES:
@@ -151,7 +150,7 @@ def get_json_response_in_bounds(self, layer: int, bounds, geometry_name: str):
                     return response.json()
                 except requests.exceptions.HTTPError:
                     if self.verbose:
-                        print(
+                        logging.info(
                             f"Layer: {layer} is not `geometry_name`: {geometry_name}."
                         )
             assert False, (
@@ -177,12 +176,10 @@ def get_features_inside_catchment(
         # properties
         features = {"geometry": []}
         for feature in feature_collection["features"]:
-
             shapely_geometry = shapely.geometry.shape(feature["geometry"])
 
             # check intersection of tile and catchment in LINZ CRS
             if self.bounding_polygon.intersects(shapely_geometry).any():
-
                 # Create column headings for each 'properties' option from the first
                 # in-bounds vector
                 if len(features["geometry"]) == 0:
@@ -193,7 +190,7 @@ def get_features_inside_catchment(
                 # Convert any one Polygon MultiPolygon to a straight Polygon then add to
                 # the geometries
                 if (
-                    shapely_geometry.geometryType() == "MultiPolygon"
+                    shapely_geometry.geom_type == "MultiPolygon"
                     and len(shapely_geometry.geoms) == 1
                 ):
                     shapely_geometry = shapely_geometry.geoms[0]
@@ -254,7 +251,6 @@ def get_features(self, layer: int) -> geopandas.GeoDataFrame:
         # properties
         features = {"geometry": []}
         for feature in feature_collection["features"]:
-
             shapely_geometry = shapely.geometry.shape(feature["geometry"])
 
             # Create column headings for each 'properties' option from the first
@@ -267,7 +263,7 @@ def get_features(self, layer: int) -> geopandas.GeoDataFrame:
             # Convert any one Polygon MultiPolygon to a straight Polygon then add to the
             # geometries
             if (
-                shapely_geometry.geometryType() == "MultiPolygon"
+                shapely_geometry.geom_type == "MultiPolygon"
                 and len(shapely_geometry.geoms) == 1
             ):
                 shapely_geometry = shapely_geometry.geoms[0]

diff --git a/tests/test_lidar/test_lidar.py b/tests/test_lidar/test_lidar.py
@@ -35,9 +35,9 @@ class OpenTopographyTest(unittest.TestCase):
     DATASETS = ["Wellington_2013", "NZ21_Kapiti"]
     FILE_SIZES = {
         "ot_CL1_WLG_2013_1km_085033.laz": 6795072,
+        "ot_CL1_WLG_2013_1km_086034.laz": 13866572,
+        "ot_CL1_WLG_2013_1km_085034.laz": 18621287,
         "ot_CL1_WLG_2013_1km_086033.laz": 5712485,
-        "ot_CL1_WLG_2013_1km_085032.laz": 1670549,
-        "ot_CL1_WLG_2013_1km_086032.laz": 72787,
         DATASETS[0] + "_TileIndex.zip": 598532,
     }
 
@@ -63,9 +63,9 @@ def setUpClass(cls):
         cls.cache_dir.mkdir()
 
         # create fake catchment boundary
-        x0 = 1764410
+        x0 = 1765600
         y0 = 5470382
-        x1 = 1765656
+        x1 = 1766200
         y1 = 5471702
         catchment = shapely.geometry.Polygon([(x0, y0), (x1, y0), (x1, y1), (x0, y1)])
         catchment = geopandas.GeoSeries([catchment])
@@ -136,7 +136,7 @@ def test_correct_wellington_files_downloaded(self):
 
         self.assertTrue(
             numpy.all([file in downloaded_files for file in dataset_dir.glob("*")]),
-            "The downloaded files {list(dataset_dir.glob('*'))} do not match the "
+            f"The downloaded files {list(dataset_dir.glob('*'))} do not match the "
             f"expected files {downloaded_files}",
         )
 

diff --git a/tests/test_lidar_subfolders/test_lidar_subfolders.py b/tests/test_lidar_subfolders/test_lidar_subfolders.py
@@ -33,29 +33,25 @@ class OpenTopographyTestSubfolders(unittest.TestCase):
     # The expected datasets and files to be downloaded - used for comparison in the
     # later tests
     DATASETS = [
-        "NZ18_Banks",
-        "NZ18_AmuriCant",
         "NZ18_Canterbury",
         "Chch_Selwn_2015",
         "Chch_Selwn_2015/NZ_Christchurch",
         "NZ20_Canterbury",
         "NZ20_Cant2",
     ]
     FILE_SIZES = {
-        DATASETS[0]: {f"{DATASETS[0]}_TileIndex.zip": 134113},
-        DATASETS[1]: {f"{DATASETS[1]}_TileIndex.zip": 813250},
-        DATASETS[2]: {
-            f"{DATASETS[2]}_TileIndex.zip": 70260,
+        DATASETS[0]: {
+            f"{DATASETS[0]}_TileIndex.zip": 70260,
             "CL2_BX24_2018_1000_2520.laz": 14829064,
         },
-        DATASETS[3]: {f"{DATASETS[3]}_TileIndex.zip": 221422},
-        DATASETS[4]: {"ot_CL2_BX24_2015_1000_2520.laz": 10761065},
-        DATASETS[5]: {
+        DATASETS[1]: {f"{DATASETS[1]}_TileIndex.zip": 221422},
+        DATASETS[2]: {"ot_CL2_BX24_2015_1000_2520.laz": 10761065},
+        DATASETS[3]: {
             "CL2_BX24_2020_1000_2520.laz": 25891330,
-            f"{DATASETS[5]}_TileIndex.zip": 120930,
+            f"{DATASETS[3]}_TileIndex.zip": 120930,
         },
-        DATASETS[6]: {
-            f"{DATASETS[6]}_TileIndex.zip": 1133609,
+        DATASETS[4]: {
+            f"{DATASETS[4]}_TileIndex.zip": 3389130,
         },
     }
 

diff --git a/tests/test_vector_linz/test_vector_linz.py b/tests/test_vector_linz/test_vector_linz.py
@@ -37,9 +37,9 @@ class LinzVectorsTest(unittest.TestCase):
         "id": [1775717, 1775718, 1775719, 1778938, 1778939],
     }
     PASTURAL_LEASE = {
-        "area": 13278778764.94074,
+        "area": 13189155932.7776,
         "geometryType": "MultiPolygon",
-        "length": 15648834.84743058,
+        "length": 15552938.848015543,
         "columns": ["geometry", "id", "lease_name"],
         "id": [12511, 12653, 12658, 12797, 12461],
     }
@@ -106,10 +106,10 @@ def compare_to_benchmark(
 
         # check various shape attributes match those expected
         self.assertEqual(
-            features.loc[0].geometry.geometryType(),
+            features.loc[0].geometry.geom_type,
             benchmark["geometryType"],
             "The geometryType of the returned {description} "
-            f"`{features.loc[0].geometry.geometryType()}` differs from the expected "
+            f"`{features.loc[0].geometry.geom_type}` differs from the expected "
             f"{benchmark['geometryType']}",
         )
         self.assertEqual(

diff --git a/tests/test_vector_linz_in_bounds/test_vector_linz_in_bounds.py b/tests/test_vector_linz_in_bounds/test_vector_linz_in_bounds.py
@@ -41,9 +41,9 @@ class LinzVectorsTest(unittest.TestCase):
 
     # The expected datasets and files to be downloaded - used for comparison in the later tests
     LAND = {
-        "area": 150539776091.31247,
+        "area": 150539779947.93246,
         "geometryType": "Polygon",
-        "length": 6002892.54900315,
+        "length": 6002542.642926594,
         "columns": [
             "geometry",
             "name",
@@ -59,7 +59,7 @@ class LinzVectorsTest(unittest.TestCase):
     BATHYMETRY_CONTOURS = {
         "area": 0.0,
         "geometryType": "LineString",
-        "length": 144353.73387463146,
+        "length": 144353.7338746315,
         "columns": [
             "geometry",
             "fidn",
@@ -74,7 +74,7 @@ class LinzVectorsTest(unittest.TestCase):
             "sorind",
             "hypcat",
         ],
-        "valdco": [2.0, 2.0, 0.0, 0.0, 0.0],
+        "valdco": [0.0, 0.0, 2.0, 0.0, 2.0],
     }
     CHATHAM_CONTOURS = None
 
@@ -152,10 +152,10 @@ def compare_to_benchmark(
 
         # check various shape attributes match those expected
         self.assertEqual(
-            features.loc[0].geometry.geometryType(),
+            features.loc[0].geometry.geom_type,
             benchmark["geometryType"],
             f"The geometryType of thereturned {description} "
-            f" `{features.loc[0].geometry.geometryType()}` differs from the expected "
+            f" `{features.loc[0].geometry.geom_type}` differs from the expected "
             f"{benchmark['geometryType']}",
         )
         self.assertEqual(

diff --git a/tests/test_vector_lris/test_vector_lris.py b/tests/test_vector_lris/test_vector_lris.py
@@ -134,10 +134,10 @@ def compare_to_benchmark(
 
         # check various shape attributes match those expected
         self.assertEqual(
-            features.loc[0].geometry.geometryType(),
+            features.loc[0].geometry.geom_type,
             benchmark["geometryType"],
             "The geometryType of the"
-            + f" returned {description} `{features.loc[0].geometry.geometryType()}` differs from the "
+            + f" returned {description} `{features.loc[0].geometry.geom_type}` differs from the "
             + f"expected {benchmark['geometryType']}",
         )
         self.assertEqual(

diff --git a/tests/test_vector_lris_in_bounds/test_vector_lris_in_bounds.py b/tests/test_vector_lris_in_bounds/test_vector_lris_in_bounds.py
@@ -164,10 +164,10 @@ def compare_to_benchmark(
 
         # check various shape attributes match those expected
         self.assertEqual(
-            features.loc[0].geometry.geometryType(),
+            features.loc[0].geometry.geom_type,
             benchmark["geometryType"],
             "The geometryType of the returned {description} "
-            f"`{features.loc[0].geometry.geometryType()}` differs from the expected "
+            f"`{features.loc[0].geometry.geom_type}` differs from the expected "
             f"{benchmark['geometryType']}",
         )
         self.assertEqual(

diff --git a/tests/test_vector_stats_nz/test_vector_stats_nz.py b/tests/test_vector_stats_nz/test_vector_stats_nz.py
@@ -126,10 +126,10 @@ def compare_to_benchmark(
 
         # check various shape attributes match those expected
         self.assertEqual(
-            features.loc[0].geometry.geometryType(),
+            features.loc[0].geometry.geom_type,
             benchmark["geometryType"],
             "The geometryType of the"
-            + f" returned {description} `{features.loc[0].geometry.geometryType()}` differs from the "
+            + f" returned {description} `{features.loc[0].geometry.geom_type}` differs from the "
             + f"expected {benchmark['geometryType']}",
         )
         self.assertEqual(

diff --git a/tests/test_vector_stats_nz_in_bounds/test_vector_stats_nz_in_bounds.py b/tests/test_vector_stats_nz_in_bounds/test_vector_stats_nz_in_bounds.py
@@ -136,10 +136,10 @@ def compare_to_benchmark(
 
         # check various shape attributes match those expected
         self.assertEqual(
-            features.loc[0].geometry.geometryType(),
+            features.loc[0].geometry.geom_type,
             benchmark["geometryType"],
             "The geometryType of the"
-            + f" returned {description} `{features.loc[0].geometry.geometryType()}` differs from the "
+            + f" returned {description} `{features.loc[0].geometry.geom_type}` differs from the "
             + f"expected {benchmark['geometryType']}",
         )
         self.assertEqual(