diff --git a/pyproject.toml b/pyproject.toml index cdaa869..e8abee3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta" [project] name = "geoapis" -version = "0.3.2" +version = "0.3.3" description = "A package for downloading geospatial data from web APIs." readme = "README.md" authors = [{ name = "Rose pearson", email = "rose.pearson@niwa.co.nz" }] @@ -27,7 +27,7 @@ dependencies = [ "boto3", "python-dotenv", "tqdm", - 'tomli; python_version < "3.11"', + 'tomli; python_version < "3.6"', ] requires-python = ">=3.6" diff --git a/src/geoapis/__init__.py b/src/geoapis/__init__.py index 9c2ac8e..e840111 100644 --- a/src/geoapis/__init__.py +++ b/src/geoapis/__init__.py @@ -4,4 +4,4 @@ @author: pearsonra """ -__version__ = "0.3.2" +__version__ = "0.3.3" diff --git a/src/geoapis/lidar.py b/src/geoapis/lidar.py index 86ca273..3c3875a 100644 --- a/src/geoapis/lidar.py +++ b/src/geoapis/lidar.py @@ -7,6 +7,7 @@ import urllib import pathlib +import logging import requests import boto3 import botocore @@ -81,11 +82,11 @@ def run(self, dataset_name: str = None): elif self.search_polygon is not None: self.download_datasets_in_polygon() else: - print( + logging.info( "Both the search_polygon and dataset_name are None. Either a " - "dataset_name of search polygon needs to be specified if any datasets " + "'dataset_name' or 'search_polygon' needs to be specified if any datasets " "are to be downloaded from OpenTopography. Please specify a " - "search_polygon during construction, or a dataset_name during run." + "'search_polygon' during construction, or a 'dataset_name' during run." ) def download_datasets_in_polygon(self): @@ -167,7 +168,7 @@ def download_dataset(self, dataset_prefix, client): dataset_prefix""" if self.verbose: - print(f"Check files in dataset {dataset_prefix}") + logging.info(f"Check files in dataset {dataset_prefix}") tile_info = self._get_dataset_tile_names(client, dataset_prefix) # check download size limit is not exceeded @@ -196,7 +197,6 @@ def _get_dataset_tile_names(self, client, dataset_prefix): # Download the file if needed if self.redownload_files_bool or not local_file_path.exists(): - # ensure folder exists before download local_file_path.parent.mkdir(parents=True, exist_ok=True) try: @@ -219,7 +219,6 @@ def _calculate_dataset_download_size(self, client, _dataset_prefix, tile_info): ) local_path = self.cache_path / file_name if self.redownload_files_bool or not local_path.exists(): - try: response = client.head_object( Bucket=self.OT_BUCKET, Key=str(file_name.as_posix()) @@ -227,7 +226,7 @@ def _calculate_dataset_download_size(self, client, _dataset_prefix, tile_info): lidar_size_bytes += response["ContentLength"] if self.verbose: - print( + logging.info( f"checking size: {file_name}: {response['ContentLength']}" f", total (GB): {self._to_gbytes(lidar_size_bytes)}" ) diff --git a/src/geoapis/vector.py b/src/geoapis/vector.py index ed22c29..6e80f19 100644 --- a/src/geoapis/vector.py +++ b/src/geoapis/vector.py @@ -6,6 +6,7 @@ """ import urllib +import logging import requests import shapely import shapely.geometry @@ -136,12 +137,10 @@ def get_json_response_in_bounds(self, layer: int, bounds, geometry_name: str): # If a geometry_name was specified use this, otherwise try the standard LINZ # ones if geometry_name is not None and geometry_name != "": - response = self.query_vector_wfs_in_bounds(layer, bounds, geometry_name) response.raise_for_status() return response.json() else: - # cycle through the standard geometry_name's - suppress errors and only # raise one if no valid responses for geometry_name in self.GEOMETRY_NAMES: @@ -151,7 +150,7 @@ def get_json_response_in_bounds(self, layer: int, bounds, geometry_name: str): return response.json() except requests.exceptions.HTTPError: if self.verbose: - print( + logging.info( f"Layer: {layer} is not `geometry_name`: {geometry_name}." ) assert False, ( @@ -177,12 +176,10 @@ def get_features_inside_catchment( # properties features = {"geometry": []} for feature in feature_collection["features"]: - shapely_geometry = shapely.geometry.shape(feature["geometry"]) # check intersection of tile and catchment in LINZ CRS if self.bounding_polygon.intersects(shapely_geometry).any(): - # Create column headings for each 'properties' option from the first # in-bounds vector if len(features["geometry"]) == 0: @@ -193,7 +190,7 @@ def get_features_inside_catchment( # Convert any one Polygon MultiPolygon to a straight Polygon then add to # the geometries if ( - shapely_geometry.geometryType() == "MultiPolygon" + shapely_geometry.geom_type == "MultiPolygon" and len(shapely_geometry.geoms) == 1 ): shapely_geometry = shapely_geometry.geoms[0] @@ -254,7 +251,6 @@ def get_features(self, layer: int) -> geopandas.GeoDataFrame: # properties features = {"geometry": []} for feature in feature_collection["features"]: - shapely_geometry = shapely.geometry.shape(feature["geometry"]) # Create column headings for each 'properties' option from the first @@ -267,7 +263,7 @@ def get_features(self, layer: int) -> geopandas.GeoDataFrame: # Convert any one Polygon MultiPolygon to a straight Polygon then add to the # geometries if ( - shapely_geometry.geometryType() == "MultiPolygon" + shapely_geometry.geom_type == "MultiPolygon" and len(shapely_geometry.geoms) == 1 ): shapely_geometry = shapely_geometry.geoms[0] diff --git a/tests/test_lidar/test_lidar.py b/tests/test_lidar/test_lidar.py index 6f2edf0..31ffe49 100644 --- a/tests/test_lidar/test_lidar.py +++ b/tests/test_lidar/test_lidar.py @@ -35,9 +35,9 @@ class OpenTopographyTest(unittest.TestCase): DATASETS = ["Wellington_2013", "NZ21_Kapiti"] FILE_SIZES = { "ot_CL1_WLG_2013_1km_085033.laz": 6795072, + "ot_CL1_WLG_2013_1km_086034.laz": 13866572, + "ot_CL1_WLG_2013_1km_085034.laz": 18621287, "ot_CL1_WLG_2013_1km_086033.laz": 5712485, - "ot_CL1_WLG_2013_1km_085032.laz": 1670549, - "ot_CL1_WLG_2013_1km_086032.laz": 72787, DATASETS[0] + "_TileIndex.zip": 598532, } @@ -63,9 +63,9 @@ def setUpClass(cls): cls.cache_dir.mkdir() # create fake catchment boundary - x0 = 1764410 + x0 = 1765600 y0 = 5470382 - x1 = 1765656 + x1 = 1766200 y1 = 5471702 catchment = shapely.geometry.Polygon([(x0, y0), (x1, y0), (x1, y1), (x0, y1)]) catchment = geopandas.GeoSeries([catchment]) @@ -136,7 +136,7 @@ def test_correct_wellington_files_downloaded(self): self.assertTrue( numpy.all([file in downloaded_files for file in dataset_dir.glob("*")]), - "The downloaded files {list(dataset_dir.glob('*'))} do not match the " + f"The downloaded files {list(dataset_dir.glob('*'))} do not match the " f"expected files {downloaded_files}", ) diff --git a/tests/test_lidar_subfolders/test_lidar_subfolders.py b/tests/test_lidar_subfolders/test_lidar_subfolders.py index cfc1378..8a3a596 100644 --- a/tests/test_lidar_subfolders/test_lidar_subfolders.py +++ b/tests/test_lidar_subfolders/test_lidar_subfolders.py @@ -33,8 +33,6 @@ class OpenTopographyTestSubfolders(unittest.TestCase): # The expected datasets and files to be downloaded - used for comparison in the # later tests DATASETS = [ - "NZ18_Banks", - "NZ18_AmuriCant", "NZ18_Canterbury", "Chch_Selwn_2015", "Chch_Selwn_2015/NZ_Christchurch", @@ -42,20 +40,18 @@ class OpenTopographyTestSubfolders(unittest.TestCase): "NZ20_Cant2", ] FILE_SIZES = { - DATASETS[0]: {f"{DATASETS[0]}_TileIndex.zip": 134113}, - DATASETS[1]: {f"{DATASETS[1]}_TileIndex.zip": 813250}, - DATASETS[2]: { - f"{DATASETS[2]}_TileIndex.zip": 70260, + DATASETS[0]: { + f"{DATASETS[0]}_TileIndex.zip": 70260, "CL2_BX24_2018_1000_2520.laz": 14829064, }, - DATASETS[3]: {f"{DATASETS[3]}_TileIndex.zip": 221422}, - DATASETS[4]: {"ot_CL2_BX24_2015_1000_2520.laz": 10761065}, - DATASETS[5]: { + DATASETS[1]: {f"{DATASETS[1]}_TileIndex.zip": 221422}, + DATASETS[2]: {"ot_CL2_BX24_2015_1000_2520.laz": 10761065}, + DATASETS[3]: { "CL2_BX24_2020_1000_2520.laz": 25891330, - f"{DATASETS[5]}_TileIndex.zip": 120930, + f"{DATASETS[3]}_TileIndex.zip": 120930, }, - DATASETS[6]: { - f"{DATASETS[6]}_TileIndex.zip": 1133609, + DATASETS[4]: { + f"{DATASETS[4]}_TileIndex.zip": 3389130, }, } diff --git a/tests/test_vector_linz/test_vector_linz.py b/tests/test_vector_linz/test_vector_linz.py index a38a41e..2c8fd8f 100644 --- a/tests/test_vector_linz/test_vector_linz.py +++ b/tests/test_vector_linz/test_vector_linz.py @@ -37,9 +37,9 @@ class LinzVectorsTest(unittest.TestCase): "id": [1775717, 1775718, 1775719, 1778938, 1778939], } PASTURAL_LEASE = { - "area": 13278778764.94074, + "area": 13189155932.7776, "geometryType": "MultiPolygon", - "length": 15648834.84743058, + "length": 15552938.848015543, "columns": ["geometry", "id", "lease_name"], "id": [12511, 12653, 12658, 12797, 12461], } @@ -106,10 +106,10 @@ def compare_to_benchmark( # check various shape attributes match those expected self.assertEqual( - features.loc[0].geometry.geometryType(), + features.loc[0].geometry.geom_type, benchmark["geometryType"], "The geometryType of the returned {description} " - f"`{features.loc[0].geometry.geometryType()}` differs from the expected " + f"`{features.loc[0].geometry.geom_type}` differs from the expected " f"{benchmark['geometryType']}", ) self.assertEqual( diff --git a/tests/test_vector_linz_in_bounds/test_vector_linz_in_bounds.py b/tests/test_vector_linz_in_bounds/test_vector_linz_in_bounds.py index 41f76fa..aba72aa 100644 --- a/tests/test_vector_linz_in_bounds/test_vector_linz_in_bounds.py +++ b/tests/test_vector_linz_in_bounds/test_vector_linz_in_bounds.py @@ -41,9 +41,9 @@ class LinzVectorsTest(unittest.TestCase): # The expected datasets and files to be downloaded - used for comparison in the later tests LAND = { - "area": 150539776091.31247, + "area": 150539779947.93246, "geometryType": "Polygon", - "length": 6002892.54900315, + "length": 6002542.642926594, "columns": [ "geometry", "name", @@ -59,7 +59,7 @@ class LinzVectorsTest(unittest.TestCase): BATHYMETRY_CONTOURS = { "area": 0.0, "geometryType": "LineString", - "length": 144353.73387463146, + "length": 144353.7338746315, "columns": [ "geometry", "fidn", @@ -74,7 +74,7 @@ class LinzVectorsTest(unittest.TestCase): "sorind", "hypcat", ], - "valdco": [2.0, 2.0, 0.0, 0.0, 0.0], + "valdco": [0.0, 0.0, 2.0, 0.0, 2.0], } CHATHAM_CONTOURS = None @@ -152,10 +152,10 @@ def compare_to_benchmark( # check various shape attributes match those expected self.assertEqual( - features.loc[0].geometry.geometryType(), + features.loc[0].geometry.geom_type, benchmark["geometryType"], f"The geometryType of thereturned {description} " - f" `{features.loc[0].geometry.geometryType()}` differs from the expected " + f" `{features.loc[0].geometry.geom_type}` differs from the expected " f"{benchmark['geometryType']}", ) self.assertEqual( diff --git a/tests/test_vector_lris/test_vector_lris.py b/tests/test_vector_lris/test_vector_lris.py index fc9331e..33734e5 100644 --- a/tests/test_vector_lris/test_vector_lris.py +++ b/tests/test_vector_lris/test_vector_lris.py @@ -134,10 +134,10 @@ def compare_to_benchmark( # check various shape attributes match those expected self.assertEqual( - features.loc[0].geometry.geometryType(), + features.loc[0].geometry.geom_type, benchmark["geometryType"], "The geometryType of the" - + f" returned {description} `{features.loc[0].geometry.geometryType()}` differs from the " + + f" returned {description} `{features.loc[0].geometry.geom_type}` differs from the " + f"expected {benchmark['geometryType']}", ) self.assertEqual( diff --git a/tests/test_vector_lris_in_bounds/test_vector_lris_in_bounds.py b/tests/test_vector_lris_in_bounds/test_vector_lris_in_bounds.py index 7149909..7eae5f3 100644 --- a/tests/test_vector_lris_in_bounds/test_vector_lris_in_bounds.py +++ b/tests/test_vector_lris_in_bounds/test_vector_lris_in_bounds.py @@ -164,10 +164,10 @@ def compare_to_benchmark( # check various shape attributes match those expected self.assertEqual( - features.loc[0].geometry.geometryType(), + features.loc[0].geometry.geom_type, benchmark["geometryType"], "The geometryType of the returned {description} " - f"`{features.loc[0].geometry.geometryType()}` differs from the expected " + f"`{features.loc[0].geometry.geom_type}` differs from the expected " f"{benchmark['geometryType']}", ) self.assertEqual( diff --git a/tests/test_vector_stats_nz/test_vector_stats_nz.py b/tests/test_vector_stats_nz/test_vector_stats_nz.py index 0613c3b..dda7b8c 100644 --- a/tests/test_vector_stats_nz/test_vector_stats_nz.py +++ b/tests/test_vector_stats_nz/test_vector_stats_nz.py @@ -126,10 +126,10 @@ def compare_to_benchmark( # check various shape attributes match those expected self.assertEqual( - features.loc[0].geometry.geometryType(), + features.loc[0].geometry.geom_type, benchmark["geometryType"], "The geometryType of the" - + f" returned {description} `{features.loc[0].geometry.geometryType()}` differs from the " + + f" returned {description} `{features.loc[0].geometry.geom_type}` differs from the " + f"expected {benchmark['geometryType']}", ) self.assertEqual( diff --git a/tests/test_vector_stats_nz_in_bounds/test_vector_stats_nz_in_bounds.py b/tests/test_vector_stats_nz_in_bounds/test_vector_stats_nz_in_bounds.py index b56f03e..2cf9a41 100644 --- a/tests/test_vector_stats_nz_in_bounds/test_vector_stats_nz_in_bounds.py +++ b/tests/test_vector_stats_nz_in_bounds/test_vector_stats_nz_in_bounds.py @@ -136,10 +136,10 @@ def compare_to_benchmark( # check various shape attributes match those expected self.assertEqual( - features.loc[0].geometry.geometryType(), + features.loc[0].geometry.geom_type, benchmark["geometryType"], "The geometryType of the" - + f" returned {description} `{features.loc[0].geometry.geometryType()}` differs from the " + + f" returned {description} `{features.loc[0].geometry.geom_type}` differs from the " + f"expected {benchmark['geometryType']}", ) self.assertEqual(