diff --git a/CHANGELOG.md b/CHANGELOG.md index 3195b633..d04b61a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ _When adding new entries to the changelog, please include issue/PR numbers where - Adds support for disabling the working-copy checkout of specific datasets using the commands `kart import DATASET --no-checkout` or `kart checkout --not-dataset=DATASET`, and re-enabling it using `kart checkout --dataset=DATASET`. [#926](https://github.com/koordinates/kart/pull/926) - Adds information on referencing and citing Kart to `CITATION`. [#914](https://github.com/koordinates/kart/pull/914) - Fixes a bug where Kart would misidentify a non-Kart repo as a Kart V1 repo in some circumstances. [#918](https://github.com/koordinates/kart/issues/918) +- Improve schema extraction for point cloud datasets. [#924](https://github.com/koordinates/kart/issues/924) ## 0.14.2 diff --git a/docs/pages/development/pointcloud_v1.rst b/docs/pages/development/pointcloud_v1.rst index 1a44f0f5..7ac78a3b 100644 --- a/docs/pages/development/pointcloud_v1.rst +++ b/docs/pages/development/pointcloud_v1.rst @@ -87,103 +87,135 @@ For example, this is the schema of a dataset using "PDRF 7": [ { "name": "X", - "dataType": "float", - "size": 64 + "dataType": "integer", + "size": 32 }, { "name": "Y", - "dataType": "float", - "size": 64 + "dataType": "integer", + "size": 32 }, { "name": "Z", - "dataType": "float", - "size": 64 + "dataType": "integer", + "size": 32 }, { "name": "Intensity", "dataType": "integer", - "size": 16 + "size": 16, + "unsigned": true }, { - "name": "ReturnNumber", + "name": "Return Number", "dataType": "integer", - "size": 8 + "size": 4, + "unsigned": true }, { - "name": "NumberOfReturns", + "name": "Number of Returns", "dataType": "integer", - "size": 8 + "size": 4, + "unsigned": true }, { - "name": "ScanDirectionFlag", + "name": "Synthetic", "dataType": "integer", - "size": 8 + "size": 1 }, { - "name": "EdgeOfFlightLine", + "name": "Key-Point", "dataType": "integer", - "size": 8 + "size": 1 }, { - "name": "Classification", + "name": "Withheld", "dataType": "integer", - "size": 8 + "size": 1 }, { - "name": "ScanAngleRank", - "dataType": "float", - "size": 32 + "name": "Overlap", + "dataType": "integer", + "size": 1 }, { - "name": "UserData", + "name": "Scanner Channel", "dataType": "integer", - "size": 8 + "size": 2, + "unsigned": true }, { - "name": "PointSourceId", + "name": "Scan Direction Flag", "dataType": "integer", - "size": 16 + "size": 1 }, { - "name": "GpsTime", - "dataType": "float", - "size": 64 + "name": "Edge of Flight Line", + "dataType": "integer", + "size": 1 }, { - "name": "ScanChannel", + "name": "Classification", "dataType": "integer", - "size": 8 + "size": 8, + "unsigned": true }, { - "name": "ClassFlags", + "name": "User Data", "dataType": "integer", - "size": 8 + "size": 8, + "unsigned": true }, { - "name": "Red", + "name": "Scan Angle", "dataType": "integer", "size": 16 }, + { + "name": "Point Source ID", + "dataType": "integer", + "size": 16, + "unsigned": true + }, + { + "name": "GPS Time", + "dataType": "float", + "size": 64 + }, + { + "name": "Red", + "dataType": "integer", + "size": 16, + "unsigned": true + }, { "name": "Green", "dataType": "integer", - "size": 16 + "size": 16, + "unsigned": true }, { "name": "Blue", "dataType": "integer", - "size": 16 + "size": 16, + "unsigned": true } ] -Kart uses `PDAL `_ internally to read and write LAS files. For certain fields, PDAL modifies the type of the field as it reads it, for either of the following reasons: -* The native type of the field is "fixed point" - for the sake of simplicity, PDAL converts these to the more widely-used floating point type. -* The native type of the field has changed over time. In order that the field can be read in a consistent way without worrying about the LAS version, PDAL converts - these fields to a type expressive enough that both old and new data can be stored in the same type. +Note: Kart vs PDAL schema extraction +#################################### + +Kart uses `PDAL `_ internally to read and write LAS files. PDAL is an abstraction layer that can read data from a variety of different +types of point cloud files, and as such, it interprets the schema in its own way to make it more interoperable with the rest of PDAL. +The schema that Kart conveys is schema of the LAS file as it is stored or specified, not as PDAL reads it, although these two concepts are very similar. Here are some differences between stored / specified schema and PDAL's interpretation: + +* Where the specification gives a dimension's name as multiple words, ie "Number of Returns", PDAL reports it in CamelCase, ie "NumberOfReturns". +* PDAL converts some dimensions which are technically stored as integers to floating point values as it applies scaling factors to them - for example, X, Y, and Z. +* Sometimes PDAL loads newer and older versions of a particular dimension in a version-independent way - ie the older 8-bit field "Scan Angle Rank" and the newer 16-bit field "Scan Angle" are both loaded as "ScanAngleRank", and both converted to floating point. -Kart exposes the schema as read by PDAL (not as it is actually stored) - all of the same changes are made. +If you need to see PDAL's interpretation of a schema instead of Kart's, you can run ``pdal info --schema ``. +A PDAL command-line executable can be found in the directory where Kart is installed. ``meta/crs.wkt`` ^^^^^^^^^^^^^^^^ diff --git a/kart/point_cloud/metadata_util.py b/kart/point_cloud/metadata_util.py index bb0e434e..4a0e1f82 100644 --- a/kart/point_cloud/metadata_util.py +++ b/kart/point_cloud/metadata_util.py @@ -1,3 +1,4 @@ +import base64 from enum import IntFlag import logging import json @@ -15,10 +16,9 @@ from kart.lfs_util import get_hash_and_size_of_file from kart.geometry import ring_as_wkt from kart.point_cloud.schema_util import ( - get_schema_from_pdrf, + get_schema_from_pdrf_and_vlr, get_record_length_from_pdrf, equivalent_copc_pdrf, - pdal_schema_to_kart_schema, ) from kart import subprocess_util as subprocess @@ -84,14 +84,23 @@ def rewrite_format(tile_metadata, rewrite_metadata=RewriteMetadata.NO_REWRITE): elif RewriteMetadata.AS_IF_CONVERTED_TO_COPC in rewrite_metadata: orig_pdrf = orig_format["pointDataRecordFormat"] new_pdrf = equivalent_copc_pdrf(orig_pdrf) - return { - "compression": "laz", - "lasVersion": "1.4", - "optimization": "copc", - "optimizationVersion": "1.0", - "pointDataRecordFormat": new_pdrf, - "pointDataRecordLength": get_record_length_from_pdrf(new_pdrf), - } + orig_length = orig_format["pointDataRecordLength"] + new_length = ( + orig_length + - get_record_length_from_pdrf(orig_pdrf) + + get_record_length_from_pdrf(new_pdrf) + ) + return _remove_nones( + { + "compression": "laz", + "lasVersion": "1.4", + "optimization": "copc", + "optimizationVersion": "1.0", + "pointDataRecordFormat": new_pdrf, + "pointDataRecordLength": new_length, + "extraBytesVlr": orig_format.get("extraBytesVlr"), + } + ) else: return orig_format @@ -103,7 +112,7 @@ def rewrite_schema(tile_metadata, rewrite_metadata=RewriteMetadata.NO_REWRITE): orig_schema = tile_metadata["schema.json"] if RewriteMetadata.AS_IF_CONVERTED_TO_COPC in rewrite_metadata: orig_pdrf = tile_metadata["format.json"]["pointDataRecordFormat"] - return get_schema_from_pdrf(equivalent_copc_pdrf(orig_pdrf)) + return get_schema_from_pdrf_and_vlr(equivalent_copc_pdrf(orig_pdrf), None) else: return orig_schema @@ -188,16 +197,20 @@ def extract_pc_tile_metadata(pc_tile_path, oid_and_size=None): compound_crs = metadata["srs"].get("compoundwkt") horizontal_crs = metadata["srs"].get("wkt") is_copc = metadata.get("copc") or False + pdrf = metadata["dataformat_id"] format_json = { "compression": "laz" if metadata["compressed"] else "las", "lasVersion": f"{metadata['major_version']}.{metadata['minor_version']}", "optimization": "copc" if is_copc else None, "optimizationVersion": get_copc_version(metadata) if is_copc else None, - "pointDataRecordFormat": metadata["dataformat_id"], + "pointDataRecordFormat": pdrf, "pointDataRecordLength": metadata["point_length"], } + extra_bytes_vlr = find_extra_bytes_vlr(metadata) + if extra_bytes_vlr: + format_json["extraBytesVlr"] = True - schema_json = pdal_schema_to_kart_schema(output["schema"]) + schema_json = get_schema_from_pdrf_and_vlr(pdrf, extra_bytes_vlr) if oid_and_size: oid, size = oid_and_size else: @@ -219,14 +232,12 @@ def extract_pc_tile_metadata(pc_tile_path, oid_and_size=None): "oid": f"sha256:{oid}", "size": size, } - if not url: - tile_info.pop("url", None) result = { "format.json": format_json, "schema.json": schema_json, "crs.wkt": normalise_wkt(compound_crs or horizontal_crs), - "tile": tile_info, + "tile": _remove_nones(tile_info), } return result @@ -260,6 +271,8 @@ def _calc_crs84_extent(src_extent, src_crs): """ Given a 3D extent with a particular CRS, return a CRS84 extent that surrounds that extent. """ + if not src_crs: + return None src_srs = osr.SpatialReference() src_srs.ImportFromWkt(src_crs) src_srs.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER) @@ -307,3 +320,19 @@ def extract_format(tile_format): if "format" in tile_format: return tile_format["format"] return tile_format + + +def find_extra_bytes_vlr(metadata): + return find_vlr(metadata, "LASF_Spec", 4) + + +def find_vlr(metadata, user_id, record_id): + for key, value in metadata.items(): + if not key.startswith("vlr"): + continue + if value["user_id"] == user_id and value["record_id"] == record_id: + return base64.b64decode(value["data"]) + + +def _remove_nones(input_dict): + return {key: value for key, value in input_dict.items() if value is not None} diff --git a/kart/point_cloud/pdal_convert.py b/kart/point_cloud/pdal_convert.py index 4a9972f1..9567d3d4 100644 --- a/kart/point_cloud/pdal_convert.py +++ b/kart/point_cloud/pdal_convert.py @@ -23,11 +23,7 @@ def convert_tile_to_copc(source, dest): "type": "readers.las", "filename": str(source), }, - { - "type": "writers.copc", - "filename": str(dest), - "forward": "all", - }, + {"type": "writers.copc", "filename": str(dest), "forward": "all"}, ] try: pdal_execute_pipeline(pipeline) @@ -54,6 +50,7 @@ def convert_tile_to_laz(source, dest, target_format): "type": "writers.las", "filename": str(dest), "forward": "all", + "extra_dims": "all", "compression": True, "major_version": major_version, "minor_version": minor_version, diff --git a/kart/point_cloud/schema_util.py b/kart/point_cloud/schema_util.py index fc57f14f..965486d2 100644 --- a/kart/point_cloud/schema_util.py +++ b/kart/point_cloud/schema_util.py @@ -1,3 +1,5 @@ +import struct + from kart.exceptions import NotYetImplemented from kart.schema import Schema @@ -18,31 +20,49 @@ # (kart) ipdb> extract_pc_tile_metadata("0.laz") PDRF0_SCHEMA = [ - {"name": "X", "dataType": "float", "size": 64}, - {"name": "Y", "dataType": "float", "size": 64}, - {"name": "Z", "dataType": "float", "size": 64}, - {"name": "Intensity", "dataType": "integer", "size": 16}, - {"name": "ReturnNumber", "dataType": "integer", "size": 8}, - {"name": "NumberOfReturns", "dataType": "integer", "size": 8}, - {"name": "ScanDirectionFlag", "dataType": "integer", "size": 8}, - {"name": "EdgeOfFlightLine", "dataType": "integer", "size": 8}, - {"name": "Classification", "dataType": "integer", "size": 8}, - {"name": "ScanAngleRank", "dataType": "float", "size": 32}, - {"name": "UserData", "dataType": "integer", "size": 8}, - {"name": "PointSourceId", "dataType": "integer", "size": 16}, + {"name": "X", "dataType": "integer", "size": 32}, + {"name": "Y", "dataType": "integer", "size": 32}, + {"name": "Z", "dataType": "integer", "size": 32}, + {"name": "Intensity", "dataType": "integer", "size": 16, "unsigned": True}, + {"name": "Return Number", "dataType": "integer", "size": 3, "unsigned": True}, + {"name": "Number of Returns", "dataType": "integer", "size": 3, "unsigned": True}, + {"name": "Scan Direction Flag", "dataType": "integer", "size": 1}, + {"name": "Edge of Flight Line", "dataType": "integer", "size": 1}, + {"name": "Classification", "dataType": "integer", "size": 5, "unsigned": True}, + {"name": "Synthetic", "dataType": "integer", "size": 1}, + {"name": "Key-Point", "dataType": "integer", "size": 1}, + {"name": "Withheld", "dataType": "integer", "size": 1}, + {"name": "Scan Angle Rank", "dataType": "integer", "size": 8}, + {"name": "User Data", "dataType": "integer", "size": 8, "unsigned": True}, + {"name": "Point Source ID", "dataType": "integer", "size": 16, "unsigned": True}, ] -GPS_TIME = {"name": "GpsTime", "dataType": "float", "size": 64} +GPS_TIME = {"name": "GPS Time", "dataType": "float", "size": 64} RED_GREEN_BLUE = [ - {"name": "Red", "dataType": "integer", "size": 16}, - {"name": "Green", "dataType": "integer", "size": 16}, - {"name": "Blue", "dataType": "integer", "size": 16}, + {"name": "Red", "dataType": "integer", "size": 16, "unsigned": True}, + {"name": "Green", "dataType": "integer", "size": 16, "unsigned": True}, + {"name": "Blue", "dataType": "integer", "size": 16, "unsigned": True}, ] -PDRF6_SCHEMA = PDRF0_SCHEMA + [ - GPS_TIME, - {"name": "ScanChannel", "dataType": "integer", "size": 8}, - {"name": "ClassFlags", "dataType": "integer", "size": 8}, +PDRF6_SCHEMA = [ + {"name": "X", "dataType": "integer", "size": 32}, + {"name": "Y", "dataType": "integer", "size": 32}, + {"name": "Z", "dataType": "integer", "size": 32}, + {"name": "Intensity", "dataType": "integer", "size": 16, "unsigned": True}, + {"name": "Return Number", "dataType": "integer", "size": 4, "unsigned": True}, + {"name": "Number of Returns", "dataType": "integer", "size": 4, "unsigned": True}, + {"name": "Synthetic", "dataType": "integer", "size": 1}, + {"name": "Key-Point", "dataType": "integer", "size": 1}, + {"name": "Withheld", "dataType": "integer", "size": 1}, + {"name": "Overlap", "dataType": "integer", "size": 1}, + {"name": "Scanner Channel", "dataType": "integer", "size": 2, "unsigned": True}, + {"name": "Scan Direction Flag", "dataType": "integer", "size": 1}, + {"name": "Edge of Flight Line", "dataType": "integer", "size": 1}, + {"name": "Classification", "dataType": "integer", "size": 8, "unsigned": True}, + {"name": "User Data", "dataType": "integer", "size": 8, "unsigned": True}, + {"name": "Scan Angle", "dataType": "integer", "size": 16}, + {"name": "Point Source ID", "dataType": "integer", "size": 16, "unsigned": True}, + {"name": "GPS Time", "dataType": "float", "size": 64}, ] INFRARED = {"name": "Infrared", "dataType": "integer", "size": 16} @@ -61,6 +81,7 @@ }.items() } +# Record length in bytes: PDRF_TO_RECORD_LENGTH = { 0: 20, 1: 28, @@ -71,8 +92,13 @@ 8: 38, } +# Make sure the schemas actually have the above sizes, otherwise there is a bug in the data above. +assert PDRF_TO_RECORD_LENGTH == { + k: sum([d["size"] for d in v]) // 8 for k, v in PDRF_TO_SCHEMA.items() +} + -def get_schema_from_pdrf(pdrf): +def get_schema_from_pdrf_and_vlr(pdrf, extra_bytes_vlr): """ Given a LAS PDRF (Point Data Record Format), get the file's schema. This schema is specified in Kart Dataset schema.json format, but the schema contents is as it @@ -80,13 +106,24 @@ def get_schema_from_pdrf(pdrf): Eg, scan angles are stored in LAS files as either integers or fixed-point numbers, but are always loaded by PDAL as floating point numbers, so that's what we put in the schema. """ - result = PDRF_TO_SCHEMA.get(pdrf) - if not result: + base_result = PDRF_TO_SCHEMA.get(pdrf) + if not base_result: # PDAL doesn't support these either: raise NotYetImplemented( "Sorry, Kart does not support point formats with waveform data (4, 5, 9 and 10)" ) - return result + if extra_bytes_vlr: + return Schema(base_result + get_schema_from_extra_bytes_vlr(extra_bytes_vlr)) + return Schema(base_result) + + +def get_schema_from_extra_bytes_vlr(extra_bytes_vlr): + result = [] + for dimension in struct.iter_unpack(" "dataType", size is measured in bits. - """ - return Schema( - [_pdal_col_schema_to_kart_col_schema(col) for col in pdal_schema["dimensions"]] - ) - - -def _pdal_col_schema_to_kart_col_schema(pdal_col_schema): - return { - "name": pdal_col_schema["name"], - "dataType": _pdal_type_to_kart_type(pdal_col_schema["type"]), - # Kart measures data-sizes in bits, PDAL in bytes. - "size": pdal_col_schema["size"] * 8, - } - - -# TODO - investigate what types PDAL can actually return - it's not the same as the LAZ spec. -# TODO - our dataset types don't have any notion of signed vs unsigned. -_PDAL_TYPE_TO_KART_TYPE = { - "floating": "float", - "unsigned": "integer", - "string": "text", +def _vlr_type_to_kart_type(vlr_datatype, options): + assert 0 <= vlr_datatype <= 10 + if vlr_datatype == 0: + return {"dataType": "blob", "length": options} + return _VLR_TYPE_TO_KART_TYPE[vlr_datatype] + + +_VLR_TYPE_TO_KART_TYPE = { + 0: {"dataType": "blob"}, + 1: {"dataType": "integer", "size": 8, "unsigned": True}, + 2: {"dataType": "integer", "size": 8}, + 3: {"dataType": "integer", "size": 16, "unsigned": True}, + 4: {"dataType": "integer", "size": 16}, + 5: {"dataType": "integer", "size": 32, "unsigned": True}, + 6: {"dataType": "integer", "size": 32}, + 7: {"dataType": "integer", "size": 64, "unsigned": True}, + 8: {"dataType": "integer", "size": 64}, + 9: {"dataType": "float", "size": 32}, + 10: {"dataType": "float", "size": 64}, } - - -def _pdal_type_to_kart_type(pdal_type): - return _PDAL_TYPE_TO_KART_TYPE.get(pdal_type) or pdal_type diff --git a/kart/point_cloud/v1.py b/kart/point_cloud/v1.py index db86c95a..51e0ea05 100644 --- a/kart/point_cloud/v1.py +++ b/kart/point_cloud/v1.py @@ -8,6 +8,7 @@ is_copc, ) from kart.point_cloud.pdal_convert import convert_tile_to_format +from kart.point_cloud.schema_util import get_schema_from_pdrf_and_vlr from kart.point_cloud.tilename_util import ( remove_tile_extension, set_tile_extension, @@ -90,6 +91,15 @@ def rewrite_and_merge_metadata( ) return rewrite_and_merge_metadata(metadata_list, rewrite_metadata) + def get_meta_item(self, meta_item_path, missing_ok=True): + if meta_item_path == "schema.json": + format_json = self.get_meta_item("format.json", missing_ok=True) + if format_json and not format_json.get("extraBytesVlr"): + return get_schema_from_pdrf_and_vlr( + format_json.get("pointDataRecordFormat"), None + ) + return super().get_meta_item(meta_item_path, missing_ok=missing_ok) + def check_merged_metadata( self, current_metadata, merged_metadata, convert_to_dataset_format=None ): diff --git a/tests/byod/test_imports.py b/tests/byod/test_imports.py index 8433dfcf..856c71a7 100644 --- a/tests/byod/test_imports.py +++ b/tests/byod/test_imports.py @@ -39,22 +39,45 @@ def test_byod_point_cloud_import( output = json.loads(r.stdout) auckland = output["kart.diff/v1+hexwkb"]["auckland"] assert auckland["meta"]["schema.json"]["+"] == [ - {"name": "X", "dataType": "float", "size": 64}, - {"name": "Y", "dataType": "float", "size": 64}, - {"name": "Z", "dataType": "float", "size": 64}, - {"name": "Intensity", "dataType": "integer", "size": 16}, - {"name": "ReturnNumber", "dataType": "integer", "size": 8}, - {"name": "NumberOfReturns", "dataType": "integer", "size": 8}, - {"name": "ScanDirectionFlag", "dataType": "integer", "size": 8}, - {"name": "EdgeOfFlightLine", "dataType": "integer", "size": 8}, - {"name": "Classification", "dataType": "integer", "size": 8}, - {"name": "ScanAngleRank", "dataType": "float", "size": 32}, - {"name": "UserData", "dataType": "integer", "size": 8}, - {"name": "PointSourceId", "dataType": "integer", "size": 16}, - {"name": "GpsTime", "dataType": "float", "size": 64}, - {"name": "Red", "dataType": "integer", "size": 16}, - {"name": "Green", "dataType": "integer", "size": 16}, - {"name": "Blue", "dataType": "integer", "size": 16}, + {"name": "X", "dataType": "integer", "size": 32}, + {"name": "Y", "dataType": "integer", "size": 32}, + {"name": "Z", "dataType": "integer", "size": 32}, + {"name": "Intensity", "dataType": "integer", "size": 16, "unsigned": True}, + { + "name": "Return Number", + "dataType": "integer", + "size": 3, + "unsigned": True, + }, + { + "name": "Number of Returns", + "dataType": "integer", + "size": 3, + "unsigned": True, + }, + {"name": "Scan Direction Flag", "dataType": "integer", "size": 1}, + {"name": "Edge of Flight Line", "dataType": "integer", "size": 1}, + { + "name": "Classification", + "dataType": "integer", + "size": 5, + "unsigned": True, + }, + {"name": "Synthetic", "dataType": "integer", "size": 1}, + {"name": "Key-Point", "dataType": "integer", "size": 1}, + {"name": "Withheld", "dataType": "integer", "size": 1}, + {"name": "Scan Angle Rank", "dataType": "integer", "size": 8}, + {"name": "User Data", "dataType": "integer", "size": 8, "unsigned": True}, + { + "name": "Point Source ID", + "dataType": "integer", + "size": 16, + "unsigned": True, + }, + {"name": "GPS Time", "dataType": "float", "size": 64}, + {"name": "Red", "dataType": "integer", "size": 16, "unsigned": True}, + {"name": "Green", "dataType": "integer", "size": 16, "unsigned": True}, + {"name": "Blue", "dataType": "integer", "size": 16, "unsigned": True}, ] tile_0_url = os.path.join( diff --git a/tests/data/point-cloud/auckland-bare.git.tgz b/tests/data/point-cloud/auckland-bare.git.tgz index cb280c64..e8f7578b 100644 Binary files a/tests/data/point-cloud/auckland-bare.git.tgz and b/tests/data/point-cloud/auckland-bare.git.tgz differ diff --git a/tests/data/point-cloud/auckland.tgz b/tests/data/point-cloud/auckland.tgz index d57b44bc..46bb342d 100644 Binary files a/tests/data/point-cloud/auckland.tgz and b/tests/data/point-cloud/auckland.tgz differ diff --git a/tests/data/point-cloud/conflicts.tgz b/tests/data/point-cloud/conflicts.tgz index 24ce6d58..4b378afe 100644 Binary files a/tests/data/point-cloud/conflicts.tgz and b/tests/data/point-cloud/conflicts.tgz differ diff --git a/tests/data/point-cloud/laz-extrabytesvlr.tgz b/tests/data/point-cloud/laz-extrabytesvlr.tgz new file mode 100644 index 00000000..f6c871db Binary files /dev/null and b/tests/data/point-cloud/laz-extrabytesvlr.tgz differ diff --git a/tests/point_cloud/test_imports.py b/tests/point_cloud/test_imports.py index f213e363..aafcb730 100644 --- a/tests/point_cloud/test_imports.py +++ b/tests/point_cloud/test_imports.py @@ -80,21 +80,67 @@ def test_import_single_las__convert( assert json.loads(r.stdout) == { "autzen": { "schema.json": [ - {"name": "X", "dataType": "float", "size": 64}, - {"name": "Y", "dataType": "float", "size": 64}, - {"name": "Z", "dataType": "float", "size": 64}, - {"name": "Intensity", "dataType": "integer", "size": 16}, - {"name": "ReturnNumber", "dataType": "integer", "size": 8}, - {"name": "NumberOfReturns", "dataType": "integer", "size": 8}, - {"name": "ScanDirectionFlag", "dataType": "integer", "size": 8}, - {"name": "EdgeOfFlightLine", "dataType": "integer", "size": 8}, - {"name": "Classification", "dataType": "integer", "size": 8}, - {"name": "ScanAngleRank", "dataType": "float", "size": 32}, - {"name": "UserData", "dataType": "integer", "size": 8}, - {"name": "PointSourceId", "dataType": "integer", "size": 16}, - {"name": "GpsTime", "dataType": "float", "size": 64}, - {"name": "ScanChannel", "dataType": "integer", "size": 8}, - {"name": "ClassFlags", "dataType": "integer", "size": 8}, + {"name": "X", "dataType": "integer", "size": 32}, + {"name": "Y", "dataType": "integer", "size": 32}, + {"name": "Z", "dataType": "integer", "size": 32}, + { + "name": "Intensity", + "dataType": "integer", + "size": 16, + "unsigned": True, + }, + { + "name": "Return Number", + "dataType": "integer", + "size": 4, + "unsigned": True, + }, + { + "name": "Number of Returns", + "dataType": "integer", + "size": 4, + "unsigned": True, + }, + {"name": "Synthetic", "dataType": "integer", "size": 1}, + {"name": "Key-Point", "dataType": "integer", "size": 1}, + {"name": "Withheld", "dataType": "integer", "size": 1}, + {"name": "Overlap", "dataType": "integer", "size": 1}, + { + "name": "Scanner Channel", + "dataType": "integer", + "size": 2, + "unsigned": True, + }, + { + "name": "Scan Direction Flag", + "dataType": "integer", + "size": 1, + }, + { + "name": "Edge of Flight Line", + "dataType": "integer", + "size": 1, + }, + { + "name": "Classification", + "dataType": "integer", + "size": 8, + "unsigned": True, + }, + { + "name": "User Data", + "dataType": "integer", + "size": 8, + "unsigned": True, + }, + {"name": "Scan Angle", "dataType": "integer", "size": 16}, + { + "name": "Point Source ID", + "dataType": "integer", + "size": 16, + "unsigned": True, + }, + {"name": "GPS Time", "dataType": "float", "size": 64}, ] } } @@ -175,24 +221,85 @@ def test_import_several_laz__convert( assert json.loads(r.stdout) == { "auckland": { "schema.json": [ - {"name": "X", "dataType": "float", "size": 64}, - {"name": "Y", "dataType": "float", "size": 64}, - {"name": "Z", "dataType": "float", "size": 64}, - {"name": "Intensity", "dataType": "integer", "size": 16}, - {"name": "ReturnNumber", "dataType": "integer", "size": 8}, - {"name": "NumberOfReturns", "dataType": "integer", "size": 8}, - {"name": "ScanDirectionFlag", "dataType": "integer", "size": 8}, - {"name": "EdgeOfFlightLine", "dataType": "integer", "size": 8}, - {"name": "Classification", "dataType": "integer", "size": 8}, - {"name": "ScanAngleRank", "dataType": "float", "size": 32}, - {"name": "UserData", "dataType": "integer", "size": 8}, - {"name": "PointSourceId", "dataType": "integer", "size": 16}, - {"name": "GpsTime", "dataType": "float", "size": 64}, - {"name": "ScanChannel", "dataType": "integer", "size": 8}, - {"name": "ClassFlags", "dataType": "integer", "size": 8}, - {"name": "Red", "dataType": "integer", "size": 16}, - {"name": "Green", "dataType": "integer", "size": 16}, - {"name": "Blue", "dataType": "integer", "size": 16}, + {"name": "X", "dataType": "integer", "size": 32}, + {"name": "Y", "dataType": "integer", "size": 32}, + {"name": "Z", "dataType": "integer", "size": 32}, + { + "name": "Intensity", + "dataType": "integer", + "size": 16, + "unsigned": True, + }, + { + "name": "Return Number", + "dataType": "integer", + "size": 4, + "unsigned": True, + }, + { + "name": "Number of Returns", + "dataType": "integer", + "size": 4, + "unsigned": True, + }, + {"name": "Synthetic", "dataType": "integer", "size": 1}, + {"name": "Key-Point", "dataType": "integer", "size": 1}, + {"name": "Withheld", "dataType": "integer", "size": 1}, + {"name": "Overlap", "dataType": "integer", "size": 1}, + { + "name": "Scanner Channel", + "dataType": "integer", + "size": 2, + "unsigned": True, + }, + { + "name": "Scan Direction Flag", + "dataType": "integer", + "size": 1, + }, + { + "name": "Edge of Flight Line", + "dataType": "integer", + "size": 1, + }, + { + "name": "Classification", + "dataType": "integer", + "size": 8, + "unsigned": True, + }, + { + "name": "User Data", + "dataType": "integer", + "size": 8, + "unsigned": True, + }, + {"name": "Scan Angle", "dataType": "integer", "size": 16}, + { + "name": "Point Source ID", + "dataType": "integer", + "size": 16, + "unsigned": True, + }, + {"name": "GPS Time", "dataType": "float", "size": 64}, + { + "name": "Red", + "dataType": "integer", + "size": 16, + "unsigned": True, + }, + { + "name": "Green", + "dataType": "integer", + "size": 16, + "unsigned": True, + }, + { + "name": "Blue", + "dataType": "integer", + "size": 16, + "unsigned": True, + }, ] } } @@ -303,7 +410,7 @@ def test_import_replace_existing( ) assert r.exit_code == 0, r.stderr - # Originally this dataset was COPC, but now it's changed to LAZ 1.2 + # Originally this dataset was COPC, but now it"s changed to LAZ 1.2 # (because we used --preserve-format) r = cli_runner.invoke(["meta", "get", "auckland", "format.json", "-ojson"]) assert r.exit_code == 0, r.stderr @@ -592,3 +699,280 @@ def test_import_convert_to_copc_mismatched_schema( # This is disallowed even though we are converting to COPC, since these tiles would have different # schemas even once converted to COPC. assert "The imported files would have more than one schema:" in r.stderr + + +def test_import_extra_bytes_vlr__no_convert( + data_archive_readonly, tmp_path, cli_runner, chdir +): + with data_archive_readonly("point-cloud/laz-extrabytesvlr.tgz") as extrabytes: + repo_path = tmp_path / "point-cloud-repo" + r = cli_runner.invoke(["init", repo_path]) + assert r.exit_code == 0 + + with chdir(repo_path): + r = cli_runner.invoke( + [ + "point-cloud-import", + f"{extrabytes}/extrabytesvlr.laz", + "--dataset-path=extrabytes", + "--preserve-format", + ] + ) + assert r.exit_code == 0, r.stderr + + r = cli_runner.invoke( + ["meta", "get", "extrabytes", "format.json", "-ojson"] + ) + assert r.exit_code == 0, r.stderr + assert json.loads(r.stdout) == { + "extrabytes": { + "format.json": { + "compression": "laz", + "lasVersion": "1.4", + "pointDataRecordFormat": 3, + "pointDataRecordLength": 61, + "extraBytesVlr": True, + } + } + } + r = cli_runner.invoke( + ["meta", "get", "extrabytes", "schema.json", "-ojson"] + ) + assert r.exit_code == 0, r.stderr + assert json.loads(r.stdout) == { + "extrabytes": { + "schema.json": [ + {"name": "X", "dataType": "integer", "size": 32}, + {"name": "Y", "dataType": "integer", "size": 32}, + {"name": "Z", "dataType": "integer", "size": 32}, + { + "name": "Intensity", + "dataType": "integer", + "size": 16, + "unsigned": True, + }, + { + "name": "Return Number", + "dataType": "integer", + "size": 3, + "unsigned": True, + }, + { + "name": "Number of Returns", + "dataType": "integer", + "size": 3, + "unsigned": True, + }, + { + "name": "Scan Direction Flag", + "dataType": "integer", + "size": 1, + }, + { + "name": "Edge of Flight Line", + "dataType": "integer", + "size": 1, + }, + { + "name": "Classification", + "dataType": "integer", + "size": 5, + "unsigned": True, + }, + {"name": "Synthetic", "dataType": "integer", "size": 1}, + {"name": "Key-Point", "dataType": "integer", "size": 1}, + {"name": "Withheld", "dataType": "integer", "size": 1}, + {"name": "Scan Angle Rank", "dataType": "integer", "size": 8}, + { + "name": "User Data", + "dataType": "integer", + "size": 8, + "unsigned": True, + }, + { + "name": "Point Source ID", + "dataType": "integer", + "size": 16, + "unsigned": True, + }, + {"name": "GPS Time", "dataType": "float", "size": 64}, + { + "name": "Red", + "dataType": "integer", + "size": 16, + "unsigned": True, + }, + { + "name": "Green", + "dataType": "integer", + "size": 16, + "unsigned": True, + }, + { + "name": "Blue", + "dataType": "integer", + "size": 16, + "unsigned": True, + }, + { + "name": "Extra Flags", + "dataType": "integer", + "size": 8, + "unsigned": True, + }, + {"name": "Temperature", "dataType": "integer", "size": 16}, + {"name": "Uptime", "dataType": "integer", "size": 32}, + { + "name": "Nanotime", + "dataType": "integer", + "size": 64, + "unsigned": True, + }, + {"name": "Gravity", "dataType": "float", "size": 32}, + {"name": "Radiosity", "dataType": "float", "size": 64}, + ] + } + } + + +def test_import_extra_bytes_vlr__convert_to_copc( + data_archive_readonly, tmp_path, cli_runner, chdir +): + with data_archive_readonly("point-cloud/laz-extrabytesvlr.tgz") as extrabytes: + repo_path = tmp_path / "point-cloud-repo" + r = cli_runner.invoke(["init", repo_path]) + assert r.exit_code == 0 + + with chdir(repo_path): + r = cli_runner.invoke( + [ + "point-cloud-import", + f"{extrabytes}/extrabytesvlr.laz", + "--dataset-path=extrabytes", + "--convert-to-copc", + ] + ) + assert r.exit_code == 0, r.stderr + + r = cli_runner.invoke( + ["meta", "get", "extrabytes", "format.json", "-ojson"] + ) + assert r.exit_code == 0, r.stderr + assert json.loads(r.stdout) == { + "extrabytes": { + "format.json": { + "compression": "laz", + "lasVersion": "1.4", + "optimization": "copc", + "optimizationVersion": "1.0", + "pointDataRecordFormat": 7, + "pointDataRecordLength": 63, + "extraBytesVlr": True, + } + } + } + r = cli_runner.invoke( + ["meta", "get", "extrabytes", "schema.json", "-ojson"] + ) + assert r.exit_code == 0, r.stderr + assert json.loads(r.stdout) == { + "extrabytes": { + "schema.json": [ + {"name": "X", "dataType": "integer", "size": 32}, + {"name": "Y", "dataType": "integer", "size": 32}, + {"name": "Z", "dataType": "integer", "size": 32}, + { + "name": "Intensity", + "dataType": "integer", + "size": 16, + "unsigned": True, + }, + { + "name": "Return Number", + "dataType": "integer", + "size": 4, + "unsigned": True, + }, + { + "name": "Number of Returns", + "dataType": "integer", + "size": 4, + "unsigned": True, + }, + {"name": "Synthetic", "dataType": "integer", "size": 1}, + {"name": "Key-Point", "dataType": "integer", "size": 1}, + {"name": "Withheld", "dataType": "integer", "size": 1}, + {"name": "Overlap", "dataType": "integer", "size": 1}, + { + "name": "Scanner Channel", + "dataType": "integer", + "size": 2, + "unsigned": True, + }, + { + "name": "Scan Direction Flag", + "dataType": "integer", + "size": 1, + }, + { + "name": "Edge of Flight Line", + "dataType": "integer", + "size": 1, + }, + { + "name": "Classification", + "dataType": "integer", + "size": 8, + "unsigned": True, + }, + { + "name": "User Data", + "dataType": "integer", + "size": 8, + "unsigned": True, + }, + {"name": "Scan Angle", "dataType": "integer", "size": 16}, + { + "name": "Point Source ID", + "dataType": "integer", + "size": 16, + "unsigned": True, + }, + {"name": "GPS Time", "dataType": "float", "size": 64}, + { + "name": "Red", + "dataType": "integer", + "size": 16, + "unsigned": True, + }, + { + "name": "Green", + "dataType": "integer", + "size": 16, + "unsigned": True, + }, + { + "name": "Blue", + "dataType": "integer", + "size": 16, + "unsigned": True, + }, + { + "name": "Extra Flags", + "dataType": "integer", + "size": 8, + "unsigned": True, + }, + {"name": "Temperature", "dataType": "integer", "size": 16}, + {"name": "Uptime", "dataType": "integer", "size": 32}, + { + "name": "Nanotime", + "dataType": "integer", + "size": 64, + "unsigned": True, + }, + {"name": "Gravity", "dataType": "float", "size": 32}, + {"name": "Radiosity", "dataType": "float", "size": 64}, + ] + } + } diff --git a/tests/point_cloud/test_workingcopy.py b/tests/point_cloud/test_workingcopy.py index 90d5a5da..5eed0e17 100644 --- a/tests/point_cloud/test_workingcopy.py +++ b/tests/point_cloud/test_workingcopy.py @@ -338,252 +338,323 @@ def test_working_copy_meta_edit( "- [", "- {", '- "name": "X",', - '- "dataType": "float",', - '- "size": 64', + '- "dataType": "integer",', + '- "size": 32', "- },", "- {", '- "name": "Y",', - '- "dataType": "float",', - '- "size": 64', + '- "dataType": "integer",', + '- "size": 32', "- },", "- {", '- "name": "Z",', - '- "dataType": "float",', - '- "size": 64', + '- "dataType": "integer",', + '- "size": 32', "- },", "- {", '- "name": "Intensity",', '- "dataType": "integer",', - '- "size": 16', + '- "size": 16,', + '- "unsigned": true', "- },", "- {", - '- "name": "ReturnNumber",', + '- "name": "Return Number",', '- "dataType": "integer",', - '- "size": 8', + '- "size": 4,', + '- "unsigned": true', "- },", "- {", - '- "name": "NumberOfReturns",', + '- "name": "Number of Returns",', '- "dataType": "integer",', - '- "size": 8', + '- "size": 4,', + '- "unsigned": true', "- },", "- {", - '- "name": "ScanDirectionFlag",', + '- "name": "Synthetic",', '- "dataType": "integer",', - '- "size": 8', + '- "size": 1', "- },", "- {", - '- "name": "EdgeOfFlightLine",', + '- "name": "Key-Point",', '- "dataType": "integer",', - '- "size": 8', + '- "size": 1', "- },", "- {", - '- "name": "Classification",', + '- "name": "Withheld",', '- "dataType": "integer",', - '- "size": 8', + '- "size": 1', "- },", "- {", - '- "name": "ScanAngleRank",', - '- "dataType": "float",', - '- "size": 32', + '- "name": "Overlap",', + '- "dataType": "integer",', + '- "size": 1', "- },", "- {", - '- "name": "UserData",', + '- "name": "Scanner Channel",', '- "dataType": "integer",', - '- "size": 8', + '- "size": 2,', + '- "unsigned": true', "- },", "- {", - '- "name": "PointSourceId",', + '- "name": "Scan Direction Flag",', '- "dataType": "integer",', - '- "size": 16', + '- "size": 1', "- },", "- {", - '- "name": "GpsTime",', - '- "dataType": "float",', - '- "size": 64', + '- "name": "Edge of Flight Line",', + '- "dataType": "integer",', + '- "size": 1', "- },", "- {", - '- "name": "ScanChannel",', + '- "name": "Classification",', '- "dataType": "integer",', - '- "size": 8', + '- "size": 8,', + '- "unsigned": true', "- },", "- {", - '- "name": "ClassFlags",', + '- "name": "User Data",', '- "dataType": "integer",', - '- "size": 8', + '- "size": 8,', + '- "unsigned": true', "- },", "- {", - '- "name": "Red",', + '- "name": "Scan Angle",', '- "dataType": "integer",', '- "size": 16', "- },", "- {", + '- "name": "Point Source ID",', + '- "dataType": "integer",', + '- "size": 16,', + '- "unsigned": true', + "- },", + "- {", + '- "name": "GPS Time",', + '- "dataType": "float",', + '- "size": 64', + "- },", + "- {", + '- "name": "Red",', + '- "dataType": "integer",', + '- "size": 16,', + '- "unsigned": true', + "- },", + "- {", '- "name": "Green",', '- "dataType": "integer",', - '- "size": 16', + '- "size": 16,', + '- "unsigned": true', "- },", "- {", '- "name": "Blue",', '- "dataType": "integer",', - '- "size": 16', + '- "size": 16,', + '- "unsigned": true', "- }", "- ]", "+ <<<<<<< ", "+ [", "+ {", '+ "name": "X",', - '+ "dataType": "float",', - '+ "size": 64', + '+ "dataType": "integer",', + '+ "size": 32', "+ },", "+ {", '+ "name": "Y",', - '+ "dataType": "float",', - '+ "size": 64', + '+ "dataType": "integer",', + '+ "size": 32', "+ },", "+ {", '+ "name": "Z",', - '+ "dataType": "float",', - '+ "size": 64', + '+ "dataType": "integer",', + '+ "size": 32', "+ },", "+ {", '+ "name": "Intensity",', '+ "dataType": "integer",', - '+ "size": 16', + '+ "size": 16,', + '+ "unsigned": true', "+ },", "+ {", - '+ "name": "ReturnNumber",', + '+ "name": "Return Number",', '+ "dataType": "integer",', - '+ "size": 8', + '+ "size": 4,', + '+ "unsigned": true', "+ },", "+ {", - '+ "name": "NumberOfReturns",', + '+ "name": "Number of Returns",', '+ "dataType": "integer",', - '+ "size": 8', + '+ "size": 4,', + '+ "unsigned": true', "+ },", "+ {", - '+ "name": "ScanDirectionFlag",', + '+ "name": "Synthetic",', '+ "dataType": "integer",', - '+ "size": 8', + '+ "size": 1', "+ },", "+ {", - '+ "name": "EdgeOfFlightLine",', + '+ "name": "Key-Point",', '+ "dataType": "integer",', - '+ "size": 8', + '+ "size": 1', "+ },", "+ {", - '+ "name": "Classification",', + '+ "name": "Withheld",', '+ "dataType": "integer",', - '+ "size": 8', + '+ "size": 1', "+ },", "+ {", - '+ "name": "ScanAngleRank",', - '+ "dataType": "float",', - '+ "size": 32', + '+ "name": "Overlap",', + '+ "dataType": "integer",', + '+ "size": 1', "+ },", "+ {", - '+ "name": "UserData",', + '+ "name": "Scanner Channel",', '+ "dataType": "integer",', - '+ "size": 8', + '+ "size": 2,', + '+ "unsigned": true', "+ },", "+ {", - '+ "name": "PointSourceId",', + '+ "name": "Scan Direction Flag",', '+ "dataType": "integer",', - '+ "size": 16', + '+ "size": 1', "+ },", "+ {", - '+ "name": "GpsTime",', - '+ "dataType": "float",', - '+ "size": 64', + '+ "name": "Edge of Flight Line",', + '+ "dataType": "integer",', + '+ "size": 1', "+ },", "+ {", - '+ "name": "ScanChannel",', + '+ "name": "Classification",', '+ "dataType": "integer",', - '+ "size": 8', + '+ "size": 8,', + '+ "unsigned": true', "+ },", "+ {", - '+ "name": "ClassFlags",', + '+ "name": "User Data",', '+ "dataType": "integer",', - '+ "size": 8', + '+ "size": 8,', + '+ "unsigned": true', "+ },", "+ {", - '+ "name": "Red",', + '+ "name": "Scan Angle",', '+ "dataType": "integer",', '+ "size": 16', "+ },", "+ {", + '+ "name": "Point Source ID",', + '+ "dataType": "integer",', + '+ "size": 16,', + '+ "unsigned": true', + "+ },", + "+ {", + '+ "name": "GPS Time",', + '+ "dataType": "float",', + '+ "size": 64', + "+ },", + "+ {", + '+ "name": "Red",', + '+ "dataType": "integer",', + '+ "size": 16,', + '+ "unsigned": true', + "+ },", + "+ {", '+ "name": "Green",', '+ "dataType": "integer",', - '+ "size": 16', + '+ "size": 16,', + '+ "unsigned": true', "+ },", "+ {", '+ "name": "Blue",', '+ "dataType": "integer",', - '+ "size": 16', + '+ "size": 16,', + '+ "unsigned": true', "+ }", "+ ]", "+ ======== ", "+ [", "+ {", '+ "name": "X",', - '+ "dataType": "float",', - '+ "size": 64', + '+ "dataType": "integer",', + '+ "size": 32', "+ },", "+ {", '+ "name": "Y",', - '+ "dataType": "float",', - '+ "size": 64', + '+ "dataType": "integer",', + '+ "size": 32', "+ },", "+ {", '+ "name": "Z",', - '+ "dataType": "float",', - '+ "size": 64', + '+ "dataType": "integer",', + '+ "size": 32', "+ },", "+ {", '+ "name": "Intensity",', '+ "dataType": "integer",', - '+ "size": 16', + '+ "size": 16,', + '+ "unsigned": true', "+ },", "+ {", - '+ "name": "ReturnNumber",', + '+ "name": "Return Number",', '+ "dataType": "integer",', - '+ "size": 8', + '+ "size": 3,', + '+ "unsigned": true', "+ },", "+ {", - '+ "name": "NumberOfReturns",', + '+ "name": "Number of Returns",', '+ "dataType": "integer",', - '+ "size": 8', + '+ "size": 3,', + '+ "unsigned": true', "+ },", "+ {", - '+ "name": "ScanDirectionFlag",', + '+ "name": "Scan Direction Flag",', '+ "dataType": "integer",', - '+ "size": 8', + '+ "size": 1', "+ },", "+ {", - '+ "name": "EdgeOfFlightLine",', + '+ "name": "Edge of Flight Line",', '+ "dataType": "integer",', - '+ "size": 8', + '+ "size": 1', "+ },", "+ {", '+ "name": "Classification",', '+ "dataType": "integer",', - '+ "size": 8', + '+ "size": 5,', + '+ "unsigned": true', "+ },", "+ {", - '+ "name": "ScanAngleRank",', - '+ "dataType": "float",', - '+ "size": 32', + '+ "name": "Synthetic",', + '+ "dataType": "integer",', + '+ "size": 1', + "+ },", + "+ {", + '+ "name": "Key-Point",', + '+ "dataType": "integer",', + '+ "size": 1', + "+ },", + "+ {", + '+ "name": "Withheld",', + '+ "dataType": "integer",', + '+ "size": 1', "+ },", "+ {", - '+ "name": "UserData",', + '+ "name": "Scan Angle Rank",', '+ "dataType": "integer",', '+ "size": 8', "+ },", "+ {", - '+ "name": "PointSourceId",', + '+ "name": "User Data",', '+ "dataType": "integer",', - '+ "size": 16', + '+ "size": 8,', + '+ "unsigned": true', + "+ },", + "+ {", + '+ "name": "Point Source ID",', + '+ "dataType": "integer",', + '+ "size": 16,', + '+ "unsigned": true', "+ },", "+ {", - '+ "name": "GpsTime",', + '+ "name": "GPS Time",', '+ "dataType": "float",', '+ "size": 64', "+ }", @@ -598,7 +669,6 @@ def test_working_copy_meta_edit( "+ oid = sha256:751ec764325610dae8f37d7f4273e3b404e5acb64421676fd72e7e31468c6720", "+ size = 2359", ] - r = cli_runner.invoke(["commit", "-m", "conflicts"]) assert r.exit_code == WORKING_COPY_OR_IMPORT_CONFLICT assert ( @@ -710,95 +780,95 @@ def test_working_copy_meta_edit( " [", " {", ' "name": "X",', - ' "dataType": "float",', - ' "size": 64', + ' "dataType": "integer",', + ' "size": 32', " },", " {", ' "name": "Y",', - ' "dataType": "float",', - ' "size": 64', - " },", - " {", - ' "name": "Z",', - ' "dataType": "float",', - ' "size": 64', - " },", - " {", - ' "name": "Intensity",', ' "dataType": "integer",', - ' "size": 16', + ' "size": 32', " },", " {", - ' "name": "ReturnNumber",', + ' "name": "Z",', ' "dataType": "integer",', - ' "size": 8', + ' "size": 32', " },", " {", - ' "name": "NumberOfReturns",', + ' "name": "Intensity",', ' "dataType": "integer",', - ' "size": 8', + ' "size": 16,', + ' "unsigned": true', " },", " {", - ' "name": "ScanDirectionFlag",', + ' "name": "Return Number",', ' "dataType": "integer",', - ' "size": 8', + '- "size": 4,', + '+ "size": 3,', + ' "unsigned": true,', " },", " {", - ' "name": "EdgeOfFlightLine",', + ' "name": "Number of Returns",', ' "dataType": "integer",', - ' "size": 8', + '- "size": 4,', + '+ "size": 3,', + ' "unsigned": true,', " },", + "+ {", + '+ "name": "Scan Direction Flag",', + '+ "dataType": "integer",', + '+ "size": 1', + "+ },", + "+ {", + '+ "name": "Edge of Flight Line",', + '+ "dataType": "integer",', + '+ "size": 1', + "+ },", + "+ {", + '+ "name": "Classification",', + '+ "dataType": "integer",', + '+ "size": 5,', + '+ "unsigned": true', + "+ },", " {", - ' "name": "Classification",', + ' "name": "Synthetic",', ' "dataType": "integer",', - ' "size": 8', + ' "size": 1', " },", " {", - ' "name": "ScanAngleRank",', - ' "dataType": "float",', - ' "size": 32', - " },", - " {", - ' "name": "UserData",', + ' "name": "Key-Point",', ' "dataType": "integer",', - ' "size": 8', + ' "size": 1', " },", " {", - ' "name": "PointSourceId",', + ' "name": "Withheld",', ' "dataType": "integer",', - ' "size": 16', - " },", - " {", - ' "name": "GpsTime",', - ' "dataType": "float",', - ' "size": 64', + ' "size": 1', " },", "- {", - '- "name": "ScanChannel",', + '- "name": "Overlap",', '- "dataType": "integer",', - '- "size": 8', + '- "size": 1', "- },", "- {", - '- "name": "ClassFlags",', + '- "name": "Scanner Channel",', '- "dataType": "integer",', - '- "size": 8', + '- "size": 2,', + '- "unsigned": true', "- },", "- {", - '- "name": "Red",', + '- "name": "Scan Direction Flag",', '- "dataType": "integer",', - '- "size": 16', + '- "size": 1', "- },", "- {", - '- "name": "Green",', + '- "name": "Edge of Flight Line",', '- "dataType": "integer",', - '- "size": 16', + '- "size": 1', "- },", "- {", - '- "name": "Blue",', + '- "name": "Classification",', '- "dataType": "integer",', - '- "size": 16', - "- },", - " ]", + '- "size": 8,', ] # We can't downgrade the dataset format from COPC to non-COPC without adding extra flags.