From 203f9238ad68992dc2396d265162cb7a2690f9fe Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 19 Jul 2024 20:36:14 -0400 Subject: [PATCH 01/19] Sketch for an MDArray implementation --- Project.toml | 4 +- src/ArchGDAL.jl | 1 + src/mdarray/mdarray.jl | 372 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 375 insertions(+), 2 deletions(-) create mode 100644 src/mdarray/mdarray.jl diff --git a/Project.toml b/Project.toml index 7b24f05e..35359498 100644 --- a/Project.toml +++ b/Project.toml @@ -22,10 +22,10 @@ Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" [compat] CEnum = "0.4, 0.5" ColorTypes = "0.10, 0.11" -Dates = "<0.0.1,1" +Dates = "<0.0.1, 1" DiskArrays = "0.3, 0.4" Extents = "0.1" -GDAL = "1.7" +GDAL = "1.8" GeoFormatTypes = "0.4.2" GeoInterface = "1" GeoInterfaceMakie = "0.1" diff --git a/src/ArchGDAL.jl b/src/ArchGDAL.jl index e7ad2ad8..766c6598 100644 --- a/src/ArchGDAL.jl +++ b/src/ArchGDAL.jl @@ -31,6 +31,7 @@ include("ogr/featurelayer.jl") include("ogr/featuredefn.jl") include("ogr/fielddefn.jl") include("ogr/styletable.jl") +include("mdarray/mdarray.jl") include("utilities.jl") include("context.jl") include("base/iterators.jl") diff --git a/src/mdarray/mdarray.jl b/src/mdarray/mdarray.jl new file mode 100644 index 00000000..3dedcd4f --- /dev/null +++ b/src/mdarray/mdarray.jl @@ -0,0 +1,372 @@ +abstract type AbstractExtendedDataType end +# needs to have a `ptr::GDALExtendedDataTypeH` attribute + +abstract type AbstractEDTComponent end +# needs to have a `ptr::GDALEDTComponentH` attribute + +abstract type AbstractGroup end +# needs to have a `ptr::GDAL.GDALGroupH` attribute + +abstract type AbstractMDArray end +# needs to have a `ptr::GDAL.GDALMDArrayH` attribute + +abstract type AbstractAttribute end +# needs to have a `ptr::GDAL.GDALAttributeH` attribute + +abstract type AbstractDimension end +# needs to have a `ptr::GDAL.GDALDimensionH` attribute + +################################################################################ + +# Question: Why do the `I...` types exist? The only difference seems +# to be that they call a finalizer. This could instead be an option to +# the constructor, simplifying the type hierarchy. + +mutable struct ExtendedDataType <: AbstractExtendedDataType + ptr::GDAL.GDALExtendedDataTypeH +end + +mutable struct EDTComponent <: AbstractEDTComponent + ptr::GDAL.GDALEDTComponentH +end + +mutable struct Group <: AbstractGroup + ptr::GDAL.GDALGroupH + + Group(ptr::GDAL.GDALGroupH = C_NULL) = new(ptr) +end + +mutable struct IGroup <: AbstractGroup + ptr::GDAL.GDALGroupH + + function IGroup(ptr::GDAL.GDALGroupH = C_NULL) + group = new(ptr) + finalizer(destroy, group) + return group + end +end + +mutable struct MDArray <: AbstractMDArray + ptr::GDAL.GDALMDArrayH + + MDArray(ptr::GDAL.GDALMDArrayH = C_NULL) = new(ptr) +end + +mutable struct IMDArray <: AbstractMDArray + ptr::GDAL.GDALMDArrayH + + function IMDArray(ptr::GDAL.GDALMDArrayH = C_NULL) + mdarray = new(ptr) + finalizer(destroy, mdarray) + return mdarray + end +end + +mutable struct Attribute <: AbstractAttribute + ptr::GDAL.GDALAttributeH + + Attribute(ptr::GDAL.GDALAttributeH = C_NULL) = new(ptr) +end + +mutable struct IAttribute <: AbstractAttribute + ptr::GDAL.GDALAttributeH + + function IAttribute(ptr::GDAL.GDALAttributeH = C_NULL) + attribute = new(ptr) + finalizer(destroy, attribute) + return attribute + end +end + +mutable struct Dimension <: AbstractDimension + ptr::GDAL.GDALDimensionH + + Dimension(ptr::GDAL.GDALDimensionH = C_NULL) = new(ptr) +end + +mutable struct IDimension <: AbstractDimension + ptr::GDAL.GDALDimensionH + + function IDimension(ptr::GDAL.GDALDimensionH = C_NULL) + dimension = new(ptr) + finalizer(destroy, dimension) + return dimension + end +end + +################################################################################ + +Base.unsafe_convert(::Type{Ptr{Cvoid}}, x::AbstractExtendedDataType) = x.ptr +Base.unsafe_convert(::Type{Ptr{Cvoid}}, x::AbstractEDTComponent) = x.ptr +Base.unsafe_convert(::Type{Ptr{Cvoid}}, x::AbstractGroup) = x.ptr +Base.unsafe_convert(::Type{Ptr{Cvoid}}, x::AbstractMDArray) = x.ptr +Base.unsafe_convert(::Type{Ptr{Cvoid}}, x::AbstractAttribute) = x.ptr +Base.unsafe_convert(::Type{Ptr{Cvoid}}, x::AbstractDimension) = x.ptr + +################################################################################ + +function destroy(datatype::AbstractExtendedDataType)::Nothing + GDAL.gdalextendeddatatyperelease(datatype) + datatype.ptr = C_NULL + return nothing +end + +function destroy(edtcomponent::AbstractEDTComponent)::Nothing + GDAL.gdaledtcomponentrelease(edtcomponent) + edtcomponent.ptr = C_NULL + return nothing +end + +function destroy(group::AbstractGroup)::Nothing + GDAL.gdalgrouprelease(group) + group.ptr = C_NULL + return nothing +end + +function destroy(mdarray::AbstractMDArray)::Nothing + GDAL.gdalmdarrayrelease(mdarray) + mdarray.ptr = C_NULL + return nothing +end + +function destroy(attribute::AbstractAttribute)::Nothing + GDAL.gdalattributerelease(attribute) + attribute.ptr = C_NULL + return nothing +end + +function destroy(dimension::AbstractDimension)::Nothing + GDAL.gdaldimensionrelease(dimension) + dimension.ptr = C_NULL + return nothing +end + +################################################################################ + +# GDALGroup + +function getname(group::AbstractGroup)::AbstractString + return GDAL.gdalgroupgetname(group) +end + +function getfullname(group::AbstractGroup)::AbstractString + return GDAL.gdalgroupgetfullname(group) +end + +function getmdarraynames( + group::AbstractGroup, + options = nothing, +)::AbstractVector{<:AbstractString} + # TODO: allow options + @assert options === nothing + return GDAL.gdalgroupgetmdarraynames(group, C_NULL) +end + +function openmdarray( + group::AbstractGroup, + name::AbstractString, + options, +)::AbstractMDArray + # TODO: allow options + @assert options === nothing + return IMDArray(GDAL.gdalgroupopenmdarray(group, name, C_NULL)) +end + +function getgroupnames( + group::AbstractGroup, + options = nothing, +)::AbstractVector{<:AbstractString} + # TODO: allow options + @assert options === nothing + return GDAL.gdalgroupgetgroupnames(group, C_NULL) +end + +function opengroup( + group::AbstractGroup, + name::AbstractString, + options, +)::AbstractGroup + # TODO: allow options + @assert options === nothing + return IGroup(GDAL.gdalgroupopengroup(group, name, C_NULL)) +end + +function getvectorlayernames( + group::AbstractGroup, + options, +)::AbstractVector{<:AbstractString} + # TODO: allow options + @assert options === nothing + return GDAL.gdalgroupgetvectorlayernames(group, C_NULL) +end + +function openvectorlayer(group::AbstractGroup, options)::AbstractFeatureLayer + # TODO: allow options + @assert options === nothing + # TODO: Find out how to set `ownedby` and `spatialref`, probably by querying `group` + return IFeatureLayer( + GDAL.openvectorlayer(group, C_NULL), + ownedby, + spatialref, + ) +end + +function getdimensions( + group::AbstractGroup, + options, +)::AbstractVector{<:AbstractDimension} + # TODO: allow options + @assert options === nothing + dimensioncountref = Ref{Csize_t}() + dimensionshptr = GDAL.gdalgroupgetdimensions(group, ndimensions, C_NULL) + dimensions = AbstractDimension[ + IDimension(unsafe_load(dimensionhptr, n)) for + n in 1:dimensionscountref[] + ] + GDAL.vsifree(dimensionshptr) + return dimensions +end + +function creategroup( + group::AbstractGroup, + name::AbstractString, + options, +)::AbstractGroup + # TODO: allow options + @assert options === nothing + return IGroup(GDAL.gdalgroupcreategroup(group, name, C_NULL)) +end + +function deletegroup(group::AbstractGroup, name::AbstractString, options)::Bool + # TODO: allow options + @assert options === nothing + return GDAL.gdalgroupdeletegroup(group, name, C_NULL) +end + +function createdimension( + group::AbstractGroup, + name::AbstractString, + type::AbstractString, + direction::AbstractString, + size::Integer, + options, +)::AbstractDimension + # TODO: allow options + @assert options === nothing + return IDimension( + GDAL.gdalgroupcreatedimension( + group, + name, + type, + direction, + size, + C_NULL, + ), + ) +end + +function createmdarray( + group::AbstractGroup, + name::AbstractString, + dimensions::AbstractVector{<:AbstractDimension}, + datatype::AbstractExtendedDataType, + options, +)::AbstractMDArray + # TODO: allow options + @assert options === nothing + dimensionhptrs = Ptr{Cvoid}[convert(Ptr{Cvoid}, dim) for dim in dimensions] + return IMDArray( + GDAL.gdalgroupcreatemdarray( + group, + name, + length(dimensionhptrs), + dimensionhptrs, + datatype, + C_NULL, + ), + ) +end + +function deletemdarray( + group::AbstractGroup, + name::AbstractString, + options, +)::Bool + # TODO: allow options + @assert options === nothing + return GDAL.gdalgroupdeletemdarray(group, name, C_NULL) +end + +# gettotalcopycost +# copyfrom + +function getstructuralinfo( + group::AbstractGroup, +)::AbstractVector{<:AbstractString} + return GDAL.gdalgroupgetstructuralinfo(group) +end + +function openmdarrayfromfullname( + group::AbstractGroup, + fullname::AbstractString, + options, +)::AbstractMDArray + # TODO: allow options + @assert options === nothing + return IMDArray( + GDAL.gdalgroupopenmdarrayfromfullname(group, fullname, C_NULL), + ) +end + +function resolvemdarray( + group::AbstractGroup, + name::AbstractString, + startingpath::AbstractString, + options, +)::AbstractMDArray + # TODO: allow options + @assert options === nothing + return IMDArray( + GDAL.gdalgroupresolvemdarray(group, name, startingpath, C_NULL), + ) +end + +function opengroupfromfullname( + group::AbstractGroup, + fullname::AbstractString, + options, +)::AbstractGroup + # TODO: allow options + @assert options === nothing + return IGroup(GDAL.gdalgroupopengroupfromfullname(group, fullname, C_NULL)) +end + +function opendimensionfromfullname( + group::AbstractGroup, + fullname::AbstractString, + options, +)::AbstractDimension + # TODO: allow options + @assert options === nothing + return IDimension( + GDAL.gdalgroupopendimensionfromfullname(group, fullname, C_NULL), + ) +end + +# clearstatistics + +function rename(group::AbstractGroup, newname::AbstractString)::Bool + return GDAL.gdalgrouprename(group, newname) +end + +function subsetdimensionfromselection( + group::AbstractGroup, + selection::AbstractString, + options, +)::AbstractGroup + # TODO: allow options + @assert options === nothing + return IGroup( + GDAL.gdalgroupsubsetdimensionfromselection(group, selection, C_NULL), + ) +end From fbc79a6a7c73d9c1969725d3e060537ccdfd68e6 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 21 Jul 2024 17:26:05 -0400 Subject: [PATCH 02/19] Make GADLGroup work --- src/context.jl | 26 ++- src/mdarray/mdarray.jl | 371 ++++++++++++++++++++++++++++++++--------- test/runtests.jl | 71 ++++---- test/test_mdarray.jl | 151 +++++++++++++++++ 4 files changed, 497 insertions(+), 122 deletions(-) create mode 100644 test/test_mdarray.jl diff --git a/src/context.jl b/src/context.jl index 364193ea..e21c2358 100644 --- a/src/context.jl +++ b/src/context.jl @@ -187,26 +187,30 @@ for gdalfunc in ( :centroid, :clone, :convexhull, + :copy, :create, + :createRAT, :createcolortable, :createcoordtrans, - :copy, + :createdimension, :createfeaturedefn, :createfielddefn, :creategeom, :creategeomcollection, - :creategeomfieldcollection, :creategeomdefn, + :creategeomfieldcollection, + :creategroup, :createlayer, :createlinearring, :createlinestring, + :createmdarray, + :createmultidimensional, :createmultilinestring, :createmultipoint, :createmultipolygon, :createmultipolygon_noholes, :createpoint, :createpolygon, - :createRAT, :createstylemanager, :createstyletable, :createstyletool, @@ -236,30 +240,38 @@ for gdalfunc in ( :getpart, :getspatialref, :importCRS, - :intersection, :importEPSG, :importEPSGA, :importESRI, :importPROJ4, + :importURL, + :importUserInput, :importWKT, :importXML, - :importUserInput, - :importURL, + :intersection, :lineargeom, :newspatialref, :nextfeature, + :opendimensionfromfullname, + :opengroup, + :opengroupfromfullname, + :openmdarray, + :openmdarrayfromfullname, + :openvectorlayer, :pointalongline, :pointonsurface, :polygonfromedges, :polygonize, :read, + :readraster, + :resolvemdarray, :sampleoverview, :simplify, :simplifypreservetopology, + :subsetdimensionfromselection, :symdifference, :union, :update, - :readraster, ) eval(quote function $(gdalfunc)(f::Function, args...; kwargs...) diff --git a/src/mdarray/mdarray.jl b/src/mdarray/mdarray.jl index 3dedcd4f..b42ecc94 100644 --- a/src/mdarray/mdarray.jl +++ b/src/mdarray/mdarray.jl @@ -7,6 +7,8 @@ abstract type AbstractEDTComponent end abstract type AbstractGroup end # needs to have a `ptr::GDAL.GDALGroupH` attribute +# TODO: Add `<: AbstractDiskArray` +# TODO: Put `{T,D}` into the type signature? abstract type AbstractMDArray end # needs to have a `ptr::GDAL.GDALMDArrayH` attribute @@ -18,12 +20,20 @@ abstract type AbstractDimension end ################################################################################ -# Question: Why do the `I...` types exist? The only difference seems -# to be that they call a finalizer. This could instead be an option to -# the constructor, simplifying the type hierarchy. - mutable struct ExtendedDataType <: AbstractExtendedDataType ptr::GDAL.GDALExtendedDataTypeH + + ExtendedDataType(ptr::GDAL.GDALExtendedDataTypeH = C_NULL) = new(ptr) +end + +mutable struct IExtendedDataType <: AbstractExtendedDataType + ptr::GDAL.GDALExtendedDataTypeH + + function IExtendedDataType(ptr::GDAL.GDALExtendedDataTypeH = C_NULL) + extendeddatatype = new(ptr) + finalizer(destroy, extendeddatatype) + return extendeddatatype + end end mutable struct EDTComponent <: AbstractEDTComponent @@ -143,6 +153,93 @@ end ################################################################################ +# Helpers + +# TODO: Move to GDAL.jl, redefining `CSLConstList` +struct CSLConstListWrapper + # Hold on to the original arguments to prevent GC from freeing + # them while they are being used in a ccall + cstrings::Vector{Cstring} + strings::AbstractVector{<:AbstractString} + + function CSLConstListWrapper( + strings::AbstractVector{<:Union{String,SubString{String}}}, + ) + cstrings = Cstring[[pointer(str) for str in strings]; C_NULL] + return new(cstrings, strings) + end +end +function CSLConstListWrapper(strings::AbstractVector{<:AbstractString}) + return String.(strings) +end + +function Base.cconvert(::Type{GDAL.CSLConstList}, wrapper::CSLConstListWrapper) + return Base.cconvert(GDAL.CSLConstList, wrapper.cstrings) +end + +struct DimensionHList + # Hold on to the original arguments to prevent GC from freeing + # them while they are being used in a ccall + dimensionhs::Vector{GDAL.GDALDimensionH} + dimensions::AbstractVector{<:AbstractDimension} + + function DimensionHList(dimensions::AbstractVector{<:AbstractDimension}) + dimensionhs = GDAL.GDALDimensionH[ + Base.unsafe_convert(Ptr{Cvoid}, dim) for dim in dimensions + ] + return new(dimensionhs, dimensions) + end +end + +function Base.cconvert( + ::Type{Ptr{GDAL.GDALDimensionH}}, + dimensionhlist::DimensionHList, +) + return Base.cconvert(Ptr{Cvoid}, dimensionhlist.dimensionhs) +end + +################################################################################ + +# Global functions + +function unsafe_createmultidimensional( + driver::Driver, + name::AbstractString, + rootgroupoptions::AbstractVector{<:AbstractString} = String[], + options::AbstractVector{<:AbstractString} = String[], +)::AbstractDataset + return Dataset( + GDAL.gdalcreatemultidimensional( + driver, + name, + CSLConstListWrapper(rootgroupoptions), + CSLConstListWrapper(options), + ), + ) +end + +function createmultidimensional( + driver::Driver, + name::AbstractString, + rootgroupoptions::AbstractVector{<:AbstractString} = String[], + options::AbstractVector{<:AbstractString} = String[], +)::AbstractDataset + return IDataset( + GDAL.gdalcreatemultidimensional( + driver, + name, + CSLConstListWrapper(rootgroupoptions), + CSLConstListWrapper(options), + ), + ) +end + +function getrootgroup(dataset::AbstractDataset)::AbstractGroup + return Group(GDAL.gdaldatasetgetrootgroup(dataset)) +end + +################################################################################ + # GDALGroup function getname(group::AbstractGroup)::AbstractString @@ -155,11 +252,17 @@ end function getmdarraynames( group::AbstractGroup, - options = nothing, + options::AbstractVector{<:AbstractString} = String[], )::AbstractVector{<:AbstractString} - # TODO: allow options - @assert options === nothing - return GDAL.gdalgroupgetmdarraynames(group, C_NULL) + return GDAL.gdalgroupgetmdarraynames(group, options) +end + +function unsafe_openmdarray( + group::AbstractGroup, + name::AbstractString, + options, +)::AbstractMDArray + return MDArray(GDAL.gdalgroupopenmdarray(group, name, options)) end function openmdarray( @@ -167,45 +270,55 @@ function openmdarray( name::AbstractString, options, )::AbstractMDArray - # TODO: allow options - @assert options === nothing - return IMDArray(GDAL.gdalgroupopenmdarray(group, name, C_NULL)) + return IMDArray(GDAL.gdalgroupopenmdarray(group, name, options)) end function getgroupnames( group::AbstractGroup, - options = nothing, + options::AbstractVector{<:AbstractString} = String[], )::AbstractVector{<:AbstractString} - # TODO: allow options - @assert options === nothing - return GDAL.gdalgroupgetgroupnames(group, C_NULL) + return GDAL.gdalgroupgetgroupnames(group, options) +end + +function unsafe_opengroup( + group::AbstractGroup, + name::AbstractString, + options::AbstractVector{<:AbstractString} = String[], +)::AbstractGroup + return Group(GDAL.gdalgroupopengroup(group, name, options)) end function opengroup( group::AbstractGroup, name::AbstractString, - options, + options::AbstractVector{<:AbstractString} = String[], )::AbstractGroup - # TODO: allow options - @assert options === nothing - return IGroup(GDAL.gdalgroupopengroup(group, name, C_NULL)) + return IGroup(GDAL.gdalgroupopengroup(group, name, options)) end function getvectorlayernames( group::AbstractGroup, - options, + options::AbstractVector{<:AbstractString} = String[], )::AbstractVector{<:AbstractString} - # TODO: allow options - @assert options === nothing - return GDAL.gdalgroupgetvectorlayernames(group, C_NULL) + return GDAL.gdalgroupgetvectorlayernames(group, options) +end + +function unsafe_openvectorlayer( + group::AbstractGroup, + options, +)::AbstractFeatureLayer + # TODO: Find out how to set `ownedby` and `spatialref`, probably by querying `group` + return FeatureLayer( + GDAL.openvectorlayer(group, options), + ownedby, + spatialref, + ) end function openvectorlayer(group::AbstractGroup, options)::AbstractFeatureLayer - # TODO: allow options - @assert options === nothing # TODO: Find out how to set `ownedby` and `spatialref`, probably by querying `group` return IFeatureLayer( - GDAL.openvectorlayer(group, C_NULL), + GDAL.openvectorlayer(group, options), ownedby, spatialref, ) @@ -213,34 +326,57 @@ end function getdimensions( group::AbstractGroup, - options, + options::AbstractVector{<:AbstractString} = String[], )::AbstractVector{<:AbstractDimension} - # TODO: allow options - @assert options === nothing - dimensioncountref = Ref{Csize_t}() - dimensionshptr = GDAL.gdalgroupgetdimensions(group, ndimensions, C_NULL) + dimensionscountref = Ref{Csize_t}() + dimensionshptr = + GDAL.gdalgroupgetdimensions(group, dimensionscountref, options) dimensions = AbstractDimension[ - IDimension(unsafe_load(dimensionhptr, n)) for + IDimension(unsafe_load(dimensionshptr, n)) for n in 1:dimensionscountref[] ] GDAL.vsifree(dimensionshptr) return dimensions end +function unsafe_creategroup( + group::AbstractGroup, + name::AbstractString, + options::AbstractVector{<:AbstractString} = String[], +)::AbstractGroup + return Group(GDAL.gdalgroupcreategroup(group, name, options)) +end + function creategroup( group::AbstractGroup, name::AbstractString, - options, + options::AbstractVector{<:AbstractString} = String[], )::AbstractGroup - # TODO: allow options - @assert options === nothing - return IGroup(GDAL.gdalgroupcreategroup(group, name, C_NULL)) + return IGroup(GDAL.gdalgroupcreategroup(group, name, options)) end function deletegroup(group::AbstractGroup, name::AbstractString, options)::Bool - # TODO: allow options - @assert options === nothing - return GDAL.gdalgroupdeletegroup(group, name, C_NULL) + return GDAL.gdalgroupdeletegroup(group, name, options) +end + +function unsafe_createdimension( + group::AbstractGroup, + name::AbstractString, + type::AbstractString, + direction::AbstractString, + size::Integer, + options::AbstractVector{<:AbstractString} = String[], +)::AbstractDimension + return Dimension( + GDAL.gdalgroupcreatedimension( + group, + name, + type, + direction, + size, + options, + ), + ) end function createdimension( @@ -249,10 +385,8 @@ function createdimension( type::AbstractString, direction::AbstractString, size::Integer, - options, + options::AbstractVector{<:AbstractString} = String[], )::AbstractDimension - # TODO: allow options - @assert options === nothing return IDimension( GDAL.gdalgroupcreatedimension( group, @@ -260,7 +394,26 @@ function createdimension( type, direction, size, - C_NULL, + options, + ), + ) +end + +function unsafe_createmdarray( + group::AbstractGroup, + name::AbstractString, + dimensions::AbstractVector{<:AbstractDimension}, + datatype::AbstractExtendedDataType, + options::AbstractVector{<:AbstractString} = String[], +)::AbstractMDArray + return MDArray( + GDAL.gdalgroupcreatemdarray( + group, + name, + length(dimensions), + DimensionHList(dimensions), + datatype, + options, ), ) end @@ -270,19 +423,16 @@ function createmdarray( name::AbstractString, dimensions::AbstractVector{<:AbstractDimension}, datatype::AbstractExtendedDataType, - options, + options::AbstractVector{<:AbstractString} = String[], )::AbstractMDArray - # TODO: allow options - @assert options === nothing - dimensionhptrs = Ptr{Cvoid}[convert(Ptr{Cvoid}, dim) for dim in dimensions] return IMDArray( GDAL.gdalgroupcreatemdarray( group, name, - length(dimensionhptrs), - dimensionhptrs, + length(dimensions), + DimensionHList(dimensions), datatype, - C_NULL, + options, ), ) end @@ -290,11 +440,9 @@ end function deletemdarray( group::AbstractGroup, name::AbstractString, - options, + options::AbstractVector{<:AbstractString} = String[], )::Bool - # TODO: allow options - @assert options === nothing - return GDAL.gdalgroupdeletemdarray(group, name, C_NULL) + return GDAL.gdalgroupdeletemdarray(group, name, options) end # gettotalcopycost @@ -306,15 +454,34 @@ function getstructuralinfo( return GDAL.gdalgroupgetstructuralinfo(group) end +function unsafe_openmdarrayfromfullname( + group::AbstractGroup, + fullname::AbstractString, + options::AbstractVector{<:AbstractString} = String[], +)::AbstractMDArray + return MDArray( + GDAL.gdalgroupopenmdarrayfromfullname(group, fullname, options), + ) +end + function openmdarrayfromfullname( group::AbstractGroup, fullname::AbstractString, - options, + options::AbstractVector{<:AbstractString} = String[], )::AbstractMDArray - # TODO: allow options - @assert options === nothing return IMDArray( - GDAL.gdalgroupopenmdarrayfromfullname(group, fullname, C_NULL), + GDAL.gdalgroupopenmdarrayfromfullname(group, fullname, options), + ) +end + +function unsafe_resolvemdarray( + group::AbstractGroup, + name::AbstractString, + startingpath::AbstractString, + options::AbstractVector{<:AbstractString} = String[], +)::AbstractMDArray + return MDArray( + GDAL.gdalgroupresolvemdarray(group, name, startingpath, options), ) end @@ -322,36 +489,48 @@ function resolvemdarray( group::AbstractGroup, name::AbstractString, startingpath::AbstractString, - options, + options::AbstractVector{<:AbstractString} = String[], )::AbstractMDArray - # TODO: allow options - @assert options === nothing return IMDArray( - GDAL.gdalgroupresolvemdarray(group, name, startingpath, C_NULL), + GDAL.gdalgroupresolvemdarray(group, name, startingpath, options), ) end -function opengroupfromfullname( +function unsafe_opengroupfromfullname( group::AbstractGroup, fullname::AbstractString, - options, + options::AbstractVector{<:AbstractString} = String[], )::AbstractGroup - # TODO: allow options - @assert options === nothing - return IGroup(GDAL.gdalgroupopengroupfromfullname(group, fullname, C_NULL)) + return Group(GDAL.gdalgroupopengroupfromfullname(group, fullname, options)) end -function opendimensionfromfullname( +function opengroupfromfullname( group::AbstractGroup, fullname::AbstractString, - options, -)::AbstractDimension - # TODO: allow options - @assert options === nothing - return IDimension( - GDAL.gdalgroupopendimensionfromfullname(group, fullname, C_NULL), - ) -end + options::AbstractVector{<:AbstractString} = String[], +)::AbstractGroup + return IGroup(GDAL.gdalgroupopengroupfromfullname(group, fullname, options)) +end + +# function unsafe_opendimensionfromfullname( +# group::AbstractGroup, +# fullname::AbstractString, +# options::AbstractVector{<:AbstractString} = String[], +# )::AbstractDimension +# return Dimension( +# GDAL.gdalgroupopendimensionfromfullname(group, fullname, options), +# ) +# end +# +# function opendimensionfromfullname( +# group::AbstractGroup, +# fullname::AbstractString, +# options::AbstractVector{<:AbstractString} = String[], +# )::AbstractDimension +# return IDimension( +# GDAL.gdalgroupopendimensionfromfullname(group, fullname, options), +# ) +# end # clearstatistics @@ -359,14 +538,46 @@ function rename(group::AbstractGroup, newname::AbstractString)::Bool return GDAL.gdalgrouprename(group, newname) end +function unsafe_subsetdimensionfromselection( + group::AbstractGroup, + selection::AbstractString, + options::AbstractVector{<:AbstractString} = String[], +)::AbstractGroup + return Group( + GDAL.gdalgroupsubsetdimensionfromselection(group, selection, options), + ) +end + function subsetdimensionfromselection( group::AbstractGroup, selection::AbstractString, - options, + options::AbstractVector{<:AbstractString} = String[], )::AbstractGroup - # TODO: allow options - @assert options === nothing return IGroup( - GDAL.gdalgroupsubsetdimensionfromselection(group, selection, C_NULL), + GDAL.gdalgroupsubsetdimensionfromselection(group, selection, options), ) end + +################################################################################ + +# GDALExtendedDataType + +function unsafe_extendeddatatypecreate( + ::Type{T}, +)::AbstractExtendedDataType where {T} + type = convert(GDALDataType, T) + return ExtendedDataType(GDAL.gdalextendeddatatypecreate(type)) +end + +function extendeddatatypecreate(::Type{T})::AbstractExtendedDataType where {T} + type = convert(GDALDataType, T) + return IExtendedDataType(GDAL.gdalextendeddatatypecreate(type)) +end + +################################################################################ + +# GDALMDArray + +function getfilename(mdarray::AbstractMDArray)::AbstractString + return GDAL.gdalmdarraygetfilename(mdarray) +end diff --git a/test/runtests.jl b/test/runtests.jl index b3aeb608..5fa1faf7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -10,41 +10,42 @@ include("remotefiles.jl") @testset "ArchGDAL" begin cd(dirname(@__FILE__)) do isdir("tmp") || mkpath("tmp") - include("test_doctest.jl") - include("test_convert.jl") - include("test_tables.jl") - include("test_gdal_tutorials.jl") - include("test_geometry.jl") - include("test_types.jl") - include("test_display.jl") - include("test_drivers.jl") - include("test_feature.jl") - include("test_featurelayer.jl") - include("test_fielddefn.jl") - include("test_iterators.jl") - include("test_styletable.jl") - include("test_dataset.jl") - include("test_rasterband.jl") - include("test_rasterio.jl") - include("test_array.jl") - include("test_spatialref.jl") - include("test_gdalutilities.jl") - include("test_gdalutilities_errors.jl") - include("test_rasterattrtable.jl") - include("test_ospy_examples.jl") - include("test_geos_operations.jl") - include("test_cookbook_geometry.jl") - include("test_cookbook_projection.jl") - include("test_geotransform.jl") - include("test_images.jl") - include("test_utils.jl") - include("test_prepared_geometry.jl") - Aqua.test_all( - ArchGDAL; - ambiguities = false, - stale_deps = false, - piracies = false, - ) + #TODO include("test_doctest.jl") + #TODO include("test_convert.jl") + #TODO include("test_tables.jl") + #TODO include("test_gdal_tutorials.jl") + #TODO include("test_geometry.jl") + #TODO include("test_types.jl") + #TODO include("test_display.jl") + #TODO include("test_drivers.jl") + #TODO include("test_feature.jl") + #TODO include("test_featurelayer.jl") + #TODO include("test_fielddefn.jl") + #TODO include("test_iterators.jl") + #TODO include("test_styletable.jl") + #TODO include("test_dataset.jl") + #TODO include("test_rasterband.jl") + #TODO include("test_rasterio.jl") + #TODO include("test_array.jl") + #TODO include("test_spatialref.jl") + #TODO include("test_gdalutilities.jl") + #TODO include("test_gdalutilities_errors.jl") + #TODO include("test_rasterattrtable.jl") + include("test_mdarray.jl") + #TODO include("test_ospy_examples.jl") + #TODO include("test_geos_operations.jl") + #TODO include("test_cookbook_geometry.jl") + #TODO include("test_cookbook_projection.jl") + #TODO include("test_geotransform.jl") + #TODO include("test_images.jl") + #TODO include("test_utils.jl") + #TODO include("test_prepared_geometry.jl") + #TODO Aqua.test_all( + #TODO ArchGDAL; + #TODO ambiguities = false, + #TODO stale_deps = false, + #TODO piracies = false, + #TODO ) return nothing end end diff --git a/test/test_mdarray.jl b/test/test_mdarray.jl new file mode 100644 index 00000000..1b250ecf --- /dev/null +++ b/test/test_mdarray.jl @@ -0,0 +1,151 @@ +using Test +import ArchGDAL as AG + +# TODO: Should this become part of ArchGDAL?u +Base.isempty(x::AG.AbstractAttribute) = x.ptr == C_NULL +Base.isempty(x::AG.AbstractDataset) = x.ptr == C_NULL +Base.isempty(x::AG.AbstractDimension) = x.ptr == C_NULL +Base.isempty(x::AG.AbstractEDTComponent) = x.ptr == C_NULL +Base.isempty(x::AG.AbstractExtendedDataType) = x.ptr == C_NULL +Base.isempty(x::AG.AbstractFeature) = x.ptr == C_NULL +Base.isempty(x::AG.AbstractFeatureDefn) = x.ptr == C_NULL +Base.isempty(x::AG.AbstractFeatureLayer) = x.ptr == C_NULL +Base.isempty(x::AG.AbstractFieldDefn) = x.ptr == C_NULL +Base.isempty(x::AG.AbstractGeomFieldDefn) = x.ptr == C_NULL +Base.isempty(x::AG.AbstractGeometry) = x.ptr == C_NULL +Base.isempty(x::AG.AbstractGroup) = x.ptr == C_NULL +Base.isempty(x::AG.AbstractMDArray) = x.ptr == C_NULL +Base.isempty(x::AG.AbstractRasterBand) = x.ptr == C_NULL +Base.isempty(x::AG.AbstractSpatialRef) = x.ptr == C_NULL +Base.isempty(x::AG.ColorTable) = x.ptr == C_NULL +Base.isempty(x::AG.CoordTransform) = x.ptr == C_NULL +Base.isempty(x::AG.Driver) = x.ptr == C_NULL +Base.isempty(x::AG.Field) = x.ptr == C_NULL +Base.isempty(x::AG.RasterAttrTable) = x.ptr == C_NULL +Base.isempty(x::AG.StyleManager) = x.ptr == C_NULL +Base.isempty(x::AG.StyleTable) = x.ptr == C_NULL +Base.isempty(x::AG.StyleTool) = x.ptr == C_NULL + +# There should be more drivers... Anyone willing to update GDAL? +# Possible drivers: at least HDF4, HDF5, TileDB +const mdarray_drivers = [ + ( + drivername = "MEM", + drivercreateoptions = String[], + mdarraycreateoptions = String[], + ), + ( + drivername = "netCDF", + drivercreateoptions = ["FORMAT=NC4"], + mdarraycreateoptions = ["COMPRESS=DEFLATE", "ZLEVEL=9"], + ), + ( + drivername = "Zarr", + drivercreateoptions = ["FORMAT=ZARR_V3"], + mdarraycreatoptions = [ + "COMPRESS=BLOSC", + "BLOSC_CLEVEL=9", + "BLOSC_SHUFFLE=BIT", + ], + ), +] + +@testset "test_mdarray.jl" begin + @testset "$drivername" for ( + drivername, + drivercreateoptions, + mdarraycreateoptions, + ) in mdarray_drivers + driver = AG.getdriver(drivername) + + filename = tempname() + dataset = AG.createmultidimensional( + driver, + filename, + String[], + drivercreateoptions, + ) + @test !isempty(dataset) + + files = AG.filelist(dataset) + if drivername in ["MEM"] + @test length(files) == 0 + elseif drivername in ["netCDF", "Zarr"] + @test length(files) == 1 + else + @assert false + end + + root = AG.getrootgroup(dataset) + @test !isempty(root) + rootname = AG.getname(root) + @test rootname == "/" + rootfullname = AG.getfullname(root) + @test rootfullname == "/" + + group = AG.creategroup(root, "group") + @test !isempty(group) + @test AG.getname(group) == "group" + @test AG.getfullname(group) == "/group" + + @test AG.getgroupnames(root) == ["group"] + @test AG.getgroupnames(group) == [] + + dimx = AG.createdimension(group, "x", "", "", 3) + @test !isempty(dimx) + dimy = AG.createdimension(group, "y", "", "", 4) + @test !isempty(dimy) + + @test AG.getdimensions(root) == [] + dimensions = AG.getdimensions(group) + @test length(dimensions) == 2 + + type = AG.extendeddatatypecreate(Float32) + @test !isempty(type) + + mdarray = AG.createmdarray( + group, + "mdarray", + [dimx, dimy], + type, + mdarraycreateoptions, + ) + @test !isempty(mdarray) + + @test AG.getmdarraynames(root) == [] + @test AG.getmdarraynames(group) == ["mdarray"] + + @test AG.getvectorlayernames(root) == [] + @test AG.getvectorlayernames(group) == [] + + @test AG.getstructuralinfo(group) == [] + + mdarray1 = AG.openmdarrayfromfullname(root, "/group/mdarray") + @test !isempty(mdarray1) + #TODO @test AG.getfullname(mdarray1) == "/group/mdarray" + @test isempty(AG.openmdarrayfromfullname(root, "/group/doesnotexist")) + + mdarray2 = AG.resolvemdarray(root, "mdarray", "") + @test !isempty(mdarray2) + #TODO @test AG.getfullname(mdarray2) == "/group/mdarray" + @test isempty(AG.resolvemdarray(root, "doesnotexist", "")) + + group1 = AG.opengroupfromfullname(root, "/group") + @test !isempty(group1) + @test AG.getfullname(group1) == "/group" + @test isempty(AG.opengroupfromfullname(root, "/doesnotexist")) + + # dimx1 = AG.opendimensionfromfullname(root, "group/x") + # @test !isempty(dimx1) + # dimz = AG.opendimensionfromfullname(root, "group/z") + # @test isempty(dimz) + + # TODO: + # - createvectorlayer + # - deletegroup + # - deletemdarary + # - openvecgtorlayer + # - rename + # - subsetdimensionfromselection + end +end From 85ba9d6533e66456e2c4c9e17eb34cdbc417780c Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 24 Jul 2024 12:37:01 -0400 Subject: [PATCH 03/19] Add MDArray support --- Project.toml | 12 +- src/ArchGDAL.jl | 4 + src/constants.jl | 42 +- src/context.jl | 16 +- src/dataset.jl | 15 +- src/mdarray/global.jl | 97 ++++ src/mdarray/group.jl | 462 ++++++++++++++++ src/mdarray/mdarray.jl | 1179 ++++++++++++++++++++++++---------------- src/mdarray/todo.jl | 13 + src/mdarray/types.jl | 282 ++++++++++ src/types.jl | 40 +- test/gdal.jl | 594 ++++++++++++++++++++ test/runtests.jl | 93 ++-- test/test_mdarray.jl | 655 ++++++++++++++++++---- 14 files changed, 2853 insertions(+), 651 deletions(-) create mode 100644 src/mdarray/global.jl create mode 100644 src/mdarray/group.jl create mode 100644 src/mdarray/todo.jl create mode 100644 src/mdarray/types.jl create mode 100644 test/gdal.jl diff --git a/Project.toml b/Project.toml index 35359498..dc07b79f 100644 --- a/Project.toml +++ b/Project.toml @@ -19,6 +19,12 @@ GeoInterfaceRecipes = "0329782f-3d07-4b52-b9f6-d3137cf03c7a" ImageCore = "a09fc81d-aa75-5fe9-8630-4744c3626534" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +[weakdeps] +Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" + +[extensions] +ArchGDALMakieExt = "Makie" + [compat] CEnum = "0.4, 0.5" ColorTypes = "0.10, 0.11" @@ -35,11 +41,5 @@ Makie = "0.20, 0.21" Tables = "1" julia = "1.6" -[extensions] -ArchGDALMakieExt = "Makie" - [extras] Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" - -[weakdeps] -Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" diff --git a/src/ArchGDAL.jl b/src/ArchGDAL.jl index 766c6598..c35e5c58 100644 --- a/src/ArchGDAL.jl +++ b/src/ArchGDAL.jl @@ -31,7 +31,11 @@ include("ogr/featurelayer.jl") include("ogr/featuredefn.jl") include("ogr/fielddefn.jl") include("ogr/styletable.jl") +include("mdarray/types.jl") +include("mdarray/global.jl") +include("mdarray/group.jl") include("mdarray/mdarray.jl") +include("mdarray/todo.jl") include("utilities.jl") include("context.jl") include("base/iterators.jl") diff --git a/src/constants.jl b/src/constants.jl index beea25af..05f01705 100644 --- a/src/constants.jl +++ b/src/constants.jl @@ -392,28 +392,30 @@ wkbXDR::OGRwkbByteOrder = 0x00000000 @enum( GDALOpenFlag, - OF_READONLY = GDAL.GDAL_OF_READONLY, # 0x00 - OF_UPDATE = GDAL.GDAL_OF_UPDATE, # 0x01 - # OF_All = GDAL.GDAL_OF_ALL, # 0x00 - OF_RASTER = GDAL.GDAL_OF_RASTER, # 0x02 - OF_VECTOR = GDAL.GDAL_OF_VECTOR, # 0x04 - OF_GNM = GDAL.GDAL_OF_GNM, # 0x08 - OF_KIND_MASK = GDAL.GDAL_OF_KIND_MASK, # 0x1e - OF_SHARED = GDAL.GDAL_OF_SHARED, # 0x20 - OF_VERBOSE_ERROR = GDAL.GDAL_OF_VERBOSE_ERROR, # 0x40 - OF_INTERNAL = GDAL.GDAL_OF_INTERNAL, # 0x80 - # OF_DEFAULT_BLOCK_ACCESS = GDAL.GDAL_OF_DEFAULT_BLOCK_ACCESS, # 0 - OF_ARRAY_BLOCK_ACCESS = GDAL.GDAL_OF_ARRAY_BLOCK_ACCESS, # 0x0100 - OF_HASHSET_BLOCK_ACCESS = GDAL.GDAL_OF_HASHSET_BLOCK_ACCESS, # 0x0200 - # OF_RESERVED_1 = GDAL.GDAL_OF_RESERVED_1, # 0x0300 - OF_BLOCK_ACCESS_MASK = GDAL.GDAL_OF_BLOCK_ACCESS_MASK, # 0x0300 + OF_READONLY = GDAL.GDAL_OF_READONLY, # 0x00 + OF_UPDATE = GDAL.GDAL_OF_UPDATE, # 0x01 + # OF_ALL = GDAL.GDAL_OF_ALL, # 0x00 + OF_RASTER = GDAL.GDAL_OF_RASTER, # 0x02 + OF_VECTOR = GDAL.GDAL_OF_VECTOR, # 0x04 + OF_GNM = GDAL.GDAL_OF_GNM, # 0x08 + OF_MULTIDIM_RASTER = GDAL.GDAL_OF_MULTIDIM_RASTER, # 0x10 + OF_KIND_MASK = GDAL.GDAL_OF_KIND_MASK, # 0x1e + OF_SHARED = GDAL.GDAL_OF_SHARED, # 0x20 + OF_VERBOSE_ERROR = GDAL.GDAL_OF_VERBOSE_ERROR, # 0x40 + OF_INTERNAL = GDAL.GDAL_OF_INTERNAL, # 0x80 + # OF_DEFAULT_BLOCK_ACCESS = GDAL.GDAL_OF_DEFAULT_BLOCK_ACCESS, # 0 + OF_ARRAY_BLOCK_ACCESS = GDAL.GDAL_OF_ARRAY_BLOCK_ACCESS, # 0x0100 + OF_HASHSET_BLOCK_ACCESS = GDAL.GDAL_OF_HASHSET_BLOCK_ACCESS, # 0x0200 + # OF_RESERVED_1 = GDAL.GDAL_OF_RESERVED_1, # 0x0300 + OF_BLOCK_ACCESS_MASK = GDAL.GDAL_OF_BLOCK_ACCESS_MASK, # 0x0300 + # OF_FROM_GDALOPEN = GDAL.GDAL_OF_FROM_GDALOPEN, # 0x0400 ) @enum( FieldValidation, - F_VAL_NULL = GDAL.OGR_F_VAL_NULL, # 0x0001 - F_VAL_GEOM_TYPE = GDAL.OGR_F_VAL_GEOM_TYPE, # 0x0002 - F_VAL_WIDTH = GDAL.OGR_F_VAL_WIDTH, # 0x0004 - F_VAL_ALLOW_NULL_WHEN_DEFAULT = GDAL.OGR_F_VAL_ALLOW_NULL_WHEN_DEFAULT, # 0x0008 - F_VAL_ALLOW_DIFFERENT_GEOM_DIM = GDAL.OGR_F_VAL_ALLOW_DIFFERENT_GEOM_DIM, # 0x0010 + F_VAL_NULL = GDAL.OGR_F_VAL_NULL, # 0x0001 + F_VAL_GEOM_TYPE = GDAL.OGR_F_VAL_GEOM_TYPE, # 0x0002 + F_VAL_WIDTH = GDAL.OGR_F_VAL_WIDTH, # 0x0004 + F_VAL_ALLOW_NULL_WHEN_DEFAULT = GDAL.OGR_F_VAL_ALLOW_NULL_WHEN_DEFAULT, # 0x0008 + F_VAL_ALLOW_DIFFERENT_GEOM_DIM = GDAL.OGR_F_VAL_ALLOW_DIFFERENT_GEOM_DIM, # 0x0010 ) diff --git a/src/context.jl b/src/context.jl index e21c2358..d720bc27 100644 --- a/src/context.jl +++ b/src/context.jl @@ -182,6 +182,8 @@ function writegeomdefn( end for gdalfunc in ( + :asclassicdataset, + :asmdarray, :boundary, :buffer, :centroid, @@ -217,6 +219,7 @@ for gdalfunc in ( :curvegeom, :delaunaytriangulation, :difference, + :extendeddatatypecreate, :forceto, :fromGML, :fromJSON, @@ -232,13 +235,22 @@ for gdalfunc in ( :gdalwarp, :getband, :getcolortable, + :getdatatype, + :getdimensions, :getfeature, :getgeom, + :getgridded, + :getindex, :getlayer, + :getmask, :getmaskband, :getoverview, :getpart, + :getresampled, + :getrootgroup, :getspatialref, + :getunscaled, + :getview, :importCRS, :importEPSG, :importEPSGA, @@ -252,7 +264,8 @@ for gdalfunc in ( :lineargeom, :newspatialref, :nextfeature, - :opendimensionfromfullname, + :open, + # :opendimensionfromfullname, :opengroup, :opengroupfromfullname, :openmdarray, @@ -270,6 +283,7 @@ for gdalfunc in ( :simplifypreservetopology, :subsetdimensionfromselection, :symdifference, + :transpose, :union, :update, ) diff --git a/src/dataset.jl b/src/dataset.jl index ba2e0d0d..ca41764f 100644 --- a/src/dataset.jl +++ b/src/dataset.jl @@ -1013,9 +1013,20 @@ function buildoverviews!( return dataset end -function destroy(dataset::AbstractDataset)::Nothing - GDAL.gdalclose(dataset) +# TODO: Wrap `GDAL.CPLErr` +function close(dataset::AbstractDataset)::GDAL.CPLErr + dataset.ptr == C_NULL && return GDAL.CE_Failure + if dataset.children !== nothing + destroy.(dataset.children) + Base.empty!(dataset.children) + end + err = GDAL.gdalclose(dataset) dataset.ptr = C_NULL + return err +end + +function destroy(dataset::AbstractDataset)::Nothing + close(dataset) return nothing end diff --git a/src/mdarray/global.jl b/src/mdarray/global.jl new file mode 100644 index 00000000..47dea33e --- /dev/null +++ b/src/mdarray/global.jl @@ -0,0 +1,97 @@ +# Global functions + +function unsafe_createmultidimensional( + driver::Driver, + name::AbstractString, + rootgroupoptions::OptionList = nothing, + options::OptionList = nothing, +)::AbstractDataset + @assert !isnull(driver) + return Dataset( + GDAL.gdalcreatemultidimensional( + driver, + name, + CSLConstListWrapper(rootgroupoptions), + CSLConstListWrapper(options), + ), + hard_close = true, + ) +end + +function createmultidimensional( + driver::Driver, + name::AbstractString, + rootgroupoptions::OptionList = nothing, + options::OptionList = nothing, +)::AbstractDataset + @assert !isnull(driver) + return IDataset( + GDAL.gdalcreatemultidimensional( + driver, + name, + CSLConstListWrapper(rootgroupoptions), + CSLConstListWrapper(options), + ), + hard_close = true, + ) +end + +function unsafe_open( + filename::AbstractString, + openflags::Integer, + alloweddrivers::OptionList, + openoptions::OptionList, + siblingfiles::OptionList, +)::AbstractDataset + # We hard-close the dataset if it is a writable multidim dataset + want_hard_close = + (openflags & OF_MULTIDIM_RASTER != 0) && (openflags & OF_UPDATE != 0) + return Dataset( + GDAL.gdalopenex( + filename, + openflags, + CSLConstListWrapper(alloweddrivers), + CSLConstListWrapper(openoptions), + CSLConstListWrapper(siblingfiles), + ), + hard_close = want_hard_close, + ) +end + +function open( + filename::AbstractString, + openflags::Integer, + alloweddrivers::OptionList, + openoptions::OptionList, + siblingfiles::OptionList, +)::AbstractDataset + # We hard-close the dataset if it is a writable multidim dataset + want_hard_close = + (openflags & OF_MULTIDIM_RASTER != 0) && (openflags & OF_UPDATE != 0) + return IDataset( + GDAL.gdalopenex( + filename, + openflags, + CSLConstListWrapper(alloweddrivers), + CSLConstListWrapper(openoptions), + CSLConstListWrapper(siblingfiles), + ), + hard_close = want_hard_close, + ) +end + +# TODO: Wrap `GDAL.CPLErr` +function flushcache!(dataset::AbstractDataset)::GDAL.CPLErr + @assert !isnull(dataset) + return GDAL.gdalflushcache(dataset) +end + +function unsafe_getrootgroup(dataset::AbstractDataset)::AbstractGroup + @assert !isnull(dataset) + return Group(GDAL.gdaldatasetgetrootgroup(dataset), dataset) +end + +function getrootgroup(dataset::AbstractDataset)::AbstractGroup + @assert !isnull(dataset) + return Group(GDAL.gdaldatasetgetrootgroup(dataset), dataset) +end diff --git a/src/mdarray/group.jl b/src/mdarray/group.jl new file mode 100644 index 00000000..83bad039 --- /dev/null +++ b/src/mdarray/group.jl @@ -0,0 +1,462 @@ +# GDALGroup + +function getname(group::AbstractGroup)::AbstractString + @assert !isnull(group) + return GDAL.gdalgroupgetname(group) +end + +function getfullname(group::AbstractGroup)::AbstractString + @assert !isnull(group) + return GDAL.gdalgroupgetfullname(group) +end + +function getmdarraynames( + group::AbstractGroup, + options::OptionList = nothing, +)::AbstractVector{<:AbstractString} + @assert !isnull(group) + return GDAL.gdalgroupgetmdarraynames(group, CSLConstListWrapper(options)) +end + +function unsafe_openmdarray( + group::AbstractGroup, + name::AbstractString, + options::OptionList = nothing, +)::AbstractMDArray + @assert !isnull(group) + return MDArray( + GDAL.gdalgroupopenmdarray(group, name, CSLConstListWrapper(options)), + group.dataset.value, + ) +end + +function openmdarray( + group::AbstractGroup, + name::AbstractString, + options::OptionList = nothing, +)::AbstractMDArray + @assert !isnull(group) + return IMDArray( + GDAL.gdalgroupopenmdarray(group, name, CSLConstListWrapper(options)), + group.dataset.value, + ) +end + +function getgroupnames( + group::AbstractGroup, + options::OptionList = nothing, +)::AbstractVector{<:AbstractString} + @assert !isnull(group) + return GDAL.gdalgroupgetgroupnames(group, CSLConstListWrapper(options)) +end + +function unsafe_opengroup( + group::AbstractGroup, + name::AbstractString, + options::OptionList = nothing, +)::AbstractGroup + @assert !isnull(group) + return Group( + GDAL.gdalgroupopengroup(group, name, CSLConstListWrapper(options)), + group.dataset.value, + ) +end + +function opengroup( + group::AbstractGroup, + name::AbstractString, + options::OptionList = nothing, +)::AbstractGroup + @assert !isnull(group) + return IGroup( + GDAL.gdalgroupopengroup(group, name, CSLConstListWrapper(options)), + group.dataset.value, + ) +end + +function getvectorlayernames( + group::AbstractGroup, + options::OptionList = nothing, +)::AbstractVector{<:AbstractString} + @assert !isnull(group) + return GDAL.gdalgroupgetvectorlayernames( + group, + CSLConstListWrapper(options), + ) +end + +function unsafe_openvectorlayer( + group::AbstractGroup, + options::OptionList = nothing, +)::AbstractFeatureLayer + @assert !isnull(group) + # TODO: Find out how to set `ownedby` and `spatialref`, probably by querying `group` + # TODO: Store dataset + return FeatureLayer( + GDAL.openvectorlayer(group, CSLConstListWrapper(options)), + ownedby, + spatialref, + ) +end + +function openvectorlayer( + group::AbstractGroup, + options::OptionList = nothing, +)::AbstractFeatureLayer + @assert !isnull(group) + # TODO: Find out how to set `ownedby` and `spatialref`, probably by querying `group` + # TODO: Store dataset + return IFeatureLayer( + GDAL.openvectorlayer(group, CSLConstListWrapper(options)), + ownedby, + spatialref, + ) +end + +function unsafe_getdimensions( + group::AbstractGroup, + options::OptionList = nothing, +)::AbstractVector{<:AbstractDimension} + @assert !isnull(group) + dimensionscountref = Ref{Csize_t}() + dimensionshptr = GDAL.gdalgroupgetdimensions( + group, + dimensionscountref, + CSLConstListWrapper(options), + ) + dataset = group.dataset.value + dimensions = AbstractDimension[ + Dimension(unsafe_load(dimensionshptr, n), dataset) for + n in 1:dimensionscountref[] + ] + GDAL.vsifree(dimensionshptr) + return dimensions +end + +function getdimensions( + group::AbstractGroup, + options::OptionList = nothing, +)::AbstractVector{<:AbstractDimension} + @assert !isnull(group) + dimensionscountref = Ref{Csize_t}() + dimensionshptr = GDAL.gdalgroupgetdimensions( + group, + dimensionscountref, + CSLConstListWrapper(options), + ) + dataset = group.dataset.value + dimensions = AbstractDimension[ + IDimension(unsafe_load(dimensionshptr, n), dataset) for + n in 1:dimensionscountref[] + ] + GDAL.vsifree(dimensionshptr) + return dimensions +end + +function unsafe_creategroup( + group::AbstractGroup, + name::AbstractString, + options::OptionList = nothing, +)::AbstractGroup + @assert !isnull(group) + return Group( + GDAL.gdalgroupcreategroup(group, name, CSLConstListWrapper(options)), + group.dataset.value, + ) +end + +function creategroup( + group::AbstractGroup, + name::AbstractString, + options::OptionList = nothing, +)::AbstractGroup + @assert !isnull(group) + return IGroup( + GDAL.gdalgroupcreategroup(group, name, CSLConstListWrapper(options)), + group.dataset.value, + ) +end + +function deletegroup( + group::AbstractGroup, + name::AbstractString, + options::OptionList = nothing, +)::Bool + @assert !isnull(group) + # TODO: Do we need to set group.ptr = C_NULL? + return GDAL.gdalgroupdeletegroup(group, name, CSLConstListWrapper(options)) +end + +function unsafe_createdimension( + group::AbstractGroup, + name::AbstractString, + type::AbstractString, + direction::AbstractString, + size::Integer, + options::OptionList = nothing, +)::AbstractDimension + @assert !isnull(group) + return Dimension( + GDAL.gdalgroupcreatedimension( + group, + name, + type, + direction, + size, + CSLConstListWrapper(options), + ), + group.dataset.value, + ) +end + +function createdimension( + group::AbstractGroup, + name::AbstractString, + type::AbstractString, + direction::AbstractString, + size::Integer, + options::OptionList = nothing, +)::AbstractDimension + @assert !isnull(group) + return IDimension( + GDAL.gdalgroupcreatedimension( + group, + name, + type, + direction, + size, + CSLConstListWrapper(options), + ), + group.dataset.value, + ) +end + +function unsafe_createmdarray( + group::AbstractGroup, + name::AbstractString, + dimensions::AbstractVector{<:AbstractDimension}, + datatype::AbstractExtendedDataType, + options::OptionList = nothing, +)::AbstractMDArray + @assert !isnull(group) + @assert all(!isnull(dim) for dim in dimensions) + @assert !isnull(datatype) + return MDArray( + GDAL.gdalgroupcreatemdarray( + group, + name, + length(dimensions), + DimensionHList(dimensions), + datatype, + CSLConstListWrapper(options), + ), + group.dataset.value, + ) +end + +function createmdarray( + group::AbstractGroup, + name::AbstractString, + dimensions::AbstractVector{<:AbstractDimension}, + datatype::AbstractExtendedDataType, + options::OptionList = nothing, +)::AbstractMDArray + @assert !isnull(group) + @assert all(!isnull(dim) for dim in dimensions) + @assert !isnull(datatype) + return IMDArray( + GDAL.gdalgroupcreatemdarray( + group, + name, + length(dimensions), + DimensionHList(dimensions), + datatype, + CSLConstListWrapper(options), + ), + group.dataset.value, + ) +end + +function deletemdarray( + group::AbstractGroup, + name::AbstractString, + options::OptionList = nothing, +)::Bool + @assert !isnull(group) + return GDAL.gdalgroupdeletemdarray( + group, + name, + CSLConstListWrapper(options), + ) +end + +# gettotalcopycost +# copyfrom + +function getstructuralinfo( + group::AbstractGroup, +)::AbstractVector{<:AbstractString} + @assert !isnull(group) + return GDAL.gdalgroupgetstructuralinfo(group) +end + +function unsafe_openmdarrayfromfullname( + group::AbstractGroup, + fullname::AbstractString, + options::OptionList = nothing, +)::AbstractMDArray + @assert !isnull(group) + return MDArray( + GDAL.gdalgroupopenmdarrayfromfullname( + group, + fullname, + CSLConstListWrapper(options), + ), + group.dataset.value, + ) +end + +function openmdarrayfromfullname( + group::AbstractGroup, + fullname::AbstractString, + options::OptionList = nothing, +)::AbstractMDArray + @assert !isnull(group) + return IMDArray( + GDAL.gdalgroupopenmdarrayfromfullname( + group, + fullname, + CSLConstListWrapper(options), + ), + group.dataset.value, + ) +end + +function unsafe_resolvemdarray( + group::AbstractGroup, + name::AbstractString, + startingpath::AbstractString, + options::OptionList = nothing, +)::AbstractMDArray + @assert !isnull(group) + return MDArray( + GDAL.gdalgroupresolvemdarray( + group, + name, + startingpath, + CSLConstListWrapper(options), + ), + group.dataset.value, + ) +end + +function resolvemdarray( + group::AbstractGroup, + name::AbstractString, + startingpath::AbstractString, + options::OptionList = nothing, +)::AbstractMDArray + @assert !isnull(group) + return IMDArray( + GDAL.gdalgroupresolvemdarray( + group, + name, + startingpath, + CSLConstListWrapper(options), + ), + group.dataset.value, + ) +end + +function unsafe_opengroupfromfullname( + group::AbstractGroup, + fullname::AbstractString, + options::OptionList = nothing, +)::AbstractGroup + @assert !isnull(group) + return Group( + GDAL.gdalgroupopengroupfromfullname( + group, + fullname, + CSLConstListWrapper(options), + ), + group.dataset.value, + ) +end + +function opengroupfromfullname( + group::AbstractGroup, + fullname::AbstractString, + options::OptionList = nothing, +)::AbstractGroup + @assert !isnull(group) + return IGroup( + GDAL.gdalgroupopengroupfromfullname( + group, + fullname, + CSLConstListWrapper(options), + ), + group.dataset.value, + ) +end + +# function unsafe_opendimensionfromfullname( +# group::AbstractGroup, +# fullname::AbstractString, +# options::OptionList=nothing, +# )::AbstractDimension +# @assert !isnull(group) +# return Dimension( +# GDAL.gdalgroupopendimensionfromfullname(group, fullname, CSLConstListWrapper(options)), group.dataset.value +# ) +# end +# +# function opendimensionfromfullname( +# group::AbstractGroup, +# fullname::AbstractString, +# options::OptionList=nothing, +# )::AbstractDimension +# @assert !isnull(group) +# return IDimension( +# GDAL.gdalgroupopendimensionfromfullname(group, fullname, CSLConstListWrapper(options)), group.dataset.value +# ) +# end + +# clearstatistics + +function rename(group::AbstractGroup, newname::AbstractString)::Bool + @assert !isnull(group) + return GDAL.gdalgrouprename(group, newname) +end + +function unsafe_subsetdimensionfromselection( + group::AbstractGroup, + selection::AbstractString, + options::OptionList = nothing, +)::AbstractGroup + @assert !isnull(group) + return Group( + GDAL.gdalgroupsubsetdimensionfromselection( + group, + selection, + CSLConstListWrapper(options), + ), + group.dataset.value, + ) +end + +function subsetdimensionfromselection( + group::AbstractGroup, + selection::AbstractString, + options::OptionList = nothing, +)::AbstractGroup + @assert !isnull(group) + return IGroup( + GDAL.gdalgroupsubsetdimensionfromselection( + group, + selection, + CSLConstListWrapper(options), + ), + group.dataset.value, + ) +end diff --git a/src/mdarray/mdarray.jl b/src/mdarray/mdarray.jl index b42ecc94..16b82611 100644 --- a/src/mdarray/mdarray.jl +++ b/src/mdarray/mdarray.jl @@ -1,583 +1,852 @@ -abstract type AbstractExtendedDataType end -# needs to have a `ptr::GDALExtendedDataTypeH` attribute - -abstract type AbstractEDTComponent end -# needs to have a `ptr::GDALEDTComponentH` attribute - -abstract type AbstractGroup end -# needs to have a `ptr::GDAL.GDALGroupH` attribute - -# TODO: Add `<: AbstractDiskArray` -# TODO: Put `{T,D}` into the type signature? -abstract type AbstractMDArray end -# needs to have a `ptr::GDAL.GDALMDArrayH` attribute - -abstract type AbstractAttribute end -# needs to have a `ptr::GDAL.GDALAttributeH` attribute - -abstract type AbstractDimension end -# needs to have a `ptr::GDAL.GDALDimensionH` attribute - -################################################################################ +# GDALMDArray -mutable struct ExtendedDataType <: AbstractExtendedDataType - ptr::GDAL.GDALExtendedDataTypeH +# function iswritable(mdarray::AbstractMDArray)::Bool +# return GDAL.gdalmdarrayiswritable(mdarray) +# end +# +# Base.iswritable(mdarray::AbstractMDArray)::Bool = iswritable(mdarray) +# Base.isreadonly(mdarray::AbstractMDArray)::Bool = !iswritable(mdarray) - ExtendedDataType(ptr::GDAL.GDALExtendedDataTypeH = C_NULL) = new(ptr) +function getfilename(mdarray::AbstractMDArray)::AbstractString + @assert !isnull(mdarray) + return GDAL.gdalmdarraygetfilename(mdarray) end -mutable struct IExtendedDataType <: AbstractExtendedDataType - ptr::GDAL.GDALExtendedDataTypeH - - function IExtendedDataType(ptr::GDAL.GDALExtendedDataTypeH = C_NULL) - extendeddatatype = new(ptr) - finalizer(destroy, extendeddatatype) - return extendeddatatype - end +function getstructuralinfo( + mdarray::AbstractMDArray, +)::AbstractVector{<:AbstractString} + @assert !isnull(mdarray) + return GDAL.gdalmdarraygetstructuralinfo(mdarray) end -mutable struct EDTComponent <: AbstractEDTComponent - ptr::GDAL.GDALEDTComponentH +function getunit(mdarray::AbstractMDArray)::AbstractString + @assert !isnull(mdarray) + return GDAL.gdalmdarraygetunit(mdarray) end -mutable struct Group <: AbstractGroup - ptr::GDAL.GDALGroupH - - Group(ptr::GDAL.GDALGroupH = C_NULL) = new(ptr) +function setunit!(mdarray::AbstractMDArray, unit::AbstractString)::Bool + @assert !isnull(mdarray) + return GDAL.gdalmdarraysetunit(mdarray, unit) end -mutable struct IGroup <: AbstractGroup - ptr::GDAL.GDALGroupH - - function IGroup(ptr::GDAL.GDALGroupH = C_NULL) - group = new(ptr) - finalizer(destroy, group) - return group - end +function setspatialref!(mdarray::AbstractMDArray, srs::AbstractSpatialRef)::Bool + @assert !isnull(mdarray) + return GDAL.gdalmdarraysetspatialref(mdarray, srs) end -mutable struct MDArray <: AbstractMDArray - ptr::GDAL.GDALMDArrayH - - MDArray(ptr::GDAL.GDALMDArrayH = C_NULL) = new(ptr) +function unsafe_getspatialref(mdarray::AbstractMDArray)::AbstractSpatialRef + @assert !isnull(mdarray) + return SpatialRef(GDAL.gdalmdarraygetspatialref(mdarray)) end -mutable struct IMDArray <: AbstractMDArray - ptr::GDAL.GDALMDArrayH - - function IMDArray(ptr::GDAL.GDALMDArrayH = C_NULL) - mdarray = new(ptr) - finalizer(destroy, mdarray) - return mdarray - end +function getspatialref(mdarray::AbstractMDArray)::AbstractSpatialRef + @assert !isnull(mdarray) + return ISpatialRef(GDAL.gdalmdarraygetspatialref(mdarray)) end -mutable struct Attribute <: AbstractAttribute - ptr::GDAL.GDALAttributeH - - Attribute(ptr::GDAL.GDALAttributeH = C_NULL) = new(ptr) +function getrawnodatavalue(mdarray::AbstractMDArray)::Ptr{Cvoid} + @assert !isnull(mdarray) + return GDAL.gdalmdarraygetrawnodatavalue(mdarray) end -mutable struct IAttribute <: AbstractAttribute - ptr::GDAL.GDALAttributeH - - function IAttribute(ptr::GDAL.GDALAttributeH = C_NULL) - attribute = new(ptr) - finalizer(destroy, attribute) - return attribute - end +function getrawnodatavalueasdouble( + mdarray::AbstractMDArray, +)::Union{Nothing,Float64} + @assert !isnull(mdarray) + hasnodata = Ref{Cbool}() + nodatavalue = GDAL.gdalmdarraygetnodatavalueasdouble(mdarray, hasnodata) + return hasnodata[] ? nodatavalue : nothing end -mutable struct Dimension <: AbstractDimension - ptr::GDAL.GDALDimensionH - - Dimension(ptr::GDAL.GDALDimensionH = C_NULL) = new(ptr) +function getrawnodatavalueasint64( + mdarray::AbstractMDArray, +)::Union{Nothing,Int64} + @assert !isnull(mdarray) + hasnodata = Ref{Cbool}() + nodatavalue = GDAL.gdalmdarraygetnodatavalueasint64(mdarray, hasnodata) + return hasnodata[] ? nodatavalue : nothing end -mutable struct IDimension <: AbstractDimension - ptr::GDAL.GDALDimensionH - - function IDimension(ptr::GDAL.GDALDimensionH = C_NULL) - dimension = new(ptr) - finalizer(destroy, dimension) - return dimension - end +function getrawnodatavalueasuint64( + mdarray::AbstractMDArray, +)::Union{Nothing,UInt64} + @assert !isnull(mdarray) + hasnodata = Ref{Cbool}() + nodatavalue = GDAL.gdalmdarraygetnodatavalueasuint64(mdarray, hasnodata) + return hasnodata[] ? nodatavalue : nothing end -################################################################################ - -Base.unsafe_convert(::Type{Ptr{Cvoid}}, x::AbstractExtendedDataType) = x.ptr -Base.unsafe_convert(::Type{Ptr{Cvoid}}, x::AbstractEDTComponent) = x.ptr -Base.unsafe_convert(::Type{Ptr{Cvoid}}, x::AbstractGroup) = x.ptr -Base.unsafe_convert(::Type{Ptr{Cvoid}}, x::AbstractMDArray) = x.ptr -Base.unsafe_convert(::Type{Ptr{Cvoid}}, x::AbstractAttribute) = x.ptr -Base.unsafe_convert(::Type{Ptr{Cvoid}}, x::AbstractDimension) = x.ptr - -################################################################################ - -function destroy(datatype::AbstractExtendedDataType)::Nothing - GDAL.gdalextendeddatatyperelease(datatype) - datatype.ptr = C_NULL - return nothing +function getnodatavalue(::Type{Float64}, mdarray::AbstractMDArray) + @assert !isnull(mdarray) + return getrawnodatavalueasdouble(mdarray) end - -function destroy(edtcomponent::AbstractEDTComponent)::Nothing - GDAL.gdaledtcomponentrelease(edtcomponent) - edtcomponent.ptr = C_NULL - return nothing -end - -function destroy(group::AbstractGroup)::Nothing - GDAL.gdalgrouprelease(group) - group.ptr = C_NULL - return nothing -end - -function destroy(mdarray::AbstractMDArray)::Nothing - GDAL.gdalmdarrayrelease(mdarray) - mdarray.ptr = C_NULL - return nothing +function getnodatavalue(::Type{Int64}, mdarray::AbstractMDArray) + @assert !isnull(mdarray) + return getrawnodatavalueasint64(mdarray) end - -function destroy(attribute::AbstractAttribute)::Nothing - GDAL.gdalattributerelease(attribute) - attribute.ptr = C_NULL - return nothing +function getnodatavalue(::Type{UInt64}, mdarray::AbstractMDArray) + @assert !isnull(mdarray) + return getrawnodatavalueasuint64(mdarray) end -function destroy(dimension::AbstractDimension)::Nothing - GDAL.gdaldimensionrelease(dimension) - dimension.ptr = C_NULL - return nothing +function setrawnodatavalue!( + mdarray::AbstractMDArray, + rawnodata::Ptr{Cvoid}, +)::Bool + @assert !isnull(mdarray) + return GDAL.gdalmdarraysetrawnodatavalue(mdarray, rawnodata) end -################################################################################ - -# Helpers - -# TODO: Move to GDAL.jl, redefining `CSLConstList` -struct CSLConstListWrapper - # Hold on to the original arguments to prevent GC from freeing - # them while they are being used in a ccall - cstrings::Vector{Cstring} - strings::AbstractVector{<:AbstractString} - - function CSLConstListWrapper( - strings::AbstractVector{<:Union{String,SubString{String}}}, - ) - cstrings = Cstring[[pointer(str) for str in strings]; C_NULL] - return new(cstrings, strings) - end -end -function CSLConstListWrapper(strings::AbstractVector{<:AbstractString}) - return String.(strings) +function setnodatavalue!(mdarray::AbstractMDArray, nodata::Float64)::Bool + @assert !isnull(mdarray) + return GDAL.gdalmdarraysetnodatavalueasdouble(mdarray, nodata) end -function Base.cconvert(::Type{GDAL.CSLConstList}, wrapper::CSLConstListWrapper) - return Base.cconvert(GDAL.CSLConstList, wrapper.cstrings) +function setnodatavalue!(mdarray::AbstractMDArray, nodata::Int64)::Bool + @assert !isnull(mdarray) + return GDAL.gdalmdarraysetnodatavalueasint64(mdarray, nodata) end -struct DimensionHList - # Hold on to the original arguments to prevent GC from freeing - # them while they are being used in a ccall - dimensionhs::Vector{GDAL.GDALDimensionH} - dimensions::AbstractVector{<:AbstractDimension} - - function DimensionHList(dimensions::AbstractVector{<:AbstractDimension}) - dimensionhs = GDAL.GDALDimensionH[ - Base.unsafe_convert(Ptr{Cvoid}, dim) for dim in dimensions - ] - return new(dimensionhs, dimensions) - end +function setnodatavalue!(mdarray::AbstractMDArray, nodata::UInt64)::Bool + @assert !isnull(mdarray) + return GDAL.gdalmdarraysetnodatavalueasuint64(mdarray, nodata) end -function Base.cconvert( - ::Type{Ptr{GDAL.GDALDimensionH}}, - dimensionhlist::DimensionHList, -) - return Base.cconvert(Ptr{Cvoid}, dimensionhlist.dimensionhs) +function resize!( + mdarray::AbstractMDArray, + newdimsizes::AbstractVector{<:Integer}, + options::OptionList = nothing, +)::Bool + @assert !isnull(mdarray) + return GDAL.gdalmdarrayresize( + mdarray, + newdimsizes, + CSLConstListWrapper(options), + ) end -################################################################################ - -# Global functions - -function unsafe_createmultidimensional( - driver::Driver, - name::AbstractString, - rootgroupoptions::AbstractVector{<:AbstractString} = String[], - options::AbstractVector{<:AbstractString} = String[], -)::AbstractDataset - return Dataset( - GDAL.gdalcreatemultidimensional( - driver, - name, - CSLConstListWrapper(rootgroupoptions), - CSLConstListWrapper(options), - ), +function getoffset(mdarray::AbstractMDArray)::Union{Nothing,Float64} + @assert !isnull(mdarray) + hasoffset = Ref{Cbool}() + offset = GDAL.gdalmdarraygetoffset(mdarray, hasoffset) + return hasoffset[] ? offset : nothing +end + +function getoffsetex( + mdarray::AbstractMDArray, +)::Union{Nothing,Tuple{Float64,Type}} + @assert !isnull(mdarray) + hasoffset = Ref{Cbool}() + storagetyperef = Ref{GDAL.GDALDataType}() + offset = GDAL.gdalmdarraygetoffsetex(mdarray, hasoffset, storagetyperef) + !hasoffset[] && return nothing + storagetype = convert(Type, storagetyperef[]) + return offset, storagetype +end + +function getscale(mdarray::AbstractMDArray)::Union{Nothing,Float64} + @assert !isnull(mdarray) + hasscale = Ref{Cbool}() + scale = GDAL.gdalmdarraygetscale(mdarray, hasscale) + return hasscale[] ? scale : nothing +end + +function getscaleex( + mdarray::AbstractMDArray, +)::Union{Nothing,Tuple{Float64,Type}} + @assert !isnull(mdarray) + hasscale = Ref{Cbool}() + storagetyperef = Ref{GDAL.GDALDataType}() + scale = GDAL.gdalmdarraygetscaleex(mdarray, hasscale, storagetyperef) + !hasscale[] && return nothing + storagetype = convert(Type, storagetyperef[]) + return scale, storagetype +end + +function setoffset!( + mdarray::AbstractMDArray, + offset::Float64, + storagetype::Union{Type,Nothing}, +)::Bool + @assert !isnull(mdarray) + return GDAL.gdalmdarraysetoffset( + mdarray, + offset, + storagetype === nothing ? GDAL.GDT_Unknown : + convert(GDAL.GDALDataType, storagetype), ) end -function createmultidimensional( - driver::Driver, - name::AbstractString, - rootgroupoptions::AbstractVector{<:AbstractString} = String[], - options::AbstractVector{<:AbstractString} = String[], -)::AbstractDataset - return IDataset( - GDAL.gdalcreatemultidimensional( - driver, - name, - CSLConstListWrapper(rootgroupoptions), - CSLConstListWrapper(options), - ), +function setscale!( + mdarray::AbstractMDArray, + offset::Float64, + storagetype::Union{Type,Nothing}, +)::Bool + @assert !isnull(mdarray) + return GDAL.gdalmdarraysetscale( + mdarray, + offset, + storagetype === nothing ? GDAL.GDT_Unknown : + convert(GDAL.GDALDataType, storagetype), ) end -function getrootgroup(dataset::AbstractDataset)::AbstractGroup - return Group(GDAL.gdaldatasetgetrootgroup(dataset)) +function unsafe_getview( + mdarray::AbstractMDArray, + viewexpr::AbstractString, +)::AbstractMDArray + @assert !isnull(mdarray) + return MDArray( + GDAL.gdalmdarraygetview(mdarray, viewexpr), + mdarray.dataset.value, + ) end -################################################################################ - -# GDALGroup - -function getname(group::AbstractGroup)::AbstractString - return GDAL.gdalgroupgetname(group) +function getview( + mdarray::AbstractMDArray, + viewexpr::AbstractString, +)::AbstractMDArray + @assert !isnull(mdarray) + return IMDArray( + GDAL.gdalmdarraygetview(mdarray, viewexpr), + mdarray.dataset.value, + ) end -function getfullname(group::AbstractGroup)::AbstractString - return GDAL.gdalgroupgetfullname(group) +function unsafe_getindex( + mdarray::AbstractMDArray, + fieldname::AbstractString, +)::AbstractMDArray + @assert !isnull(mdarray) + viewexpr = "['" * replace(fieldname, '\\' => "\\\\", '\'' => "\\\'") * "']" + return unsafe_getview(mdarray, viewexpr) end -function getmdarraynames( - group::AbstractGroup, - options::AbstractVector{<:AbstractString} = String[], -)::AbstractVector{<:AbstractString} - return GDAL.gdalgroupgetmdarraynames(group, options) +function getindex( + mdarray::AbstractMDArray, + fieldname::AbstractString, +)::AbstractMDArray + @assert !isnull(mdarray) + viewexpr = "['" * replace(fieldname, '\\' => "\\\\", '\'' => "\\\'") * "']" + return getview(mdarray, viewexpr) end -function unsafe_openmdarray( - group::AbstractGroup, - name::AbstractString, - options, +function unsafe_getindex( + mdarray::AbstractMDArray, + indices::Integer..., )::AbstractMDArray - return MDArray(GDAL.gdalgroupopenmdarray(group, name, options)) + @assert !isnull(mdarray) + viewexpr = "[" * join(indices, ",") * "]" + return unsafe_getview(mdarray, viewexpr) end -function openmdarray( - group::AbstractGroup, - name::AbstractString, - options, +function getindex( + mdarray::AbstractMDArray, + indices::Integer..., )::AbstractMDArray - return IMDArray(GDAL.gdalgroupopenmdarray(group, name, options)) + @assert !isnull(mdarray) + viewexpr = "[" * join(indices, ",") * "]" + return getview(mdarray, viewexpr) end -function getgroupnames( - group::AbstractGroup, - options::AbstractVector{<:AbstractString} = String[], -)::AbstractVector{<:AbstractString} - return GDAL.gdalgroupgetgroupnames(group, options) +# TODO: Return a `LinearAlgebra.Adjoint` instead? +function unsafe_transpose(mdarray::AbstractMDArray)::AbstractMDArray + @assert !isnull(mdarray) + return MDArray(GDAL.gdalmdarraytranspose(mdarray), mdarray.dataset.value) end -function unsafe_opengroup( - group::AbstractGroup, - name::AbstractString, - options::AbstractVector{<:AbstractString} = String[], -)::AbstractGroup - return Group(GDAL.gdalgroupopengroup(group, name, options)) +function transpose(mdarray::AbstractMDArray)::AbstractMDArray + @assert !isnull(mdarray) + return IMDArray(GDAL.gdalmdarraytranspose(mdarray), mdarray.dataset.value) end -function opengroup( - group::AbstractGroup, - name::AbstractString, - options::AbstractVector{<:AbstractString} = String[], -)::AbstractGroup - return IGroup(GDAL.gdalgroupopengroup(group, name, options)) +function unsafe_getunscaled( + mdarray::AbstractMDArray, + overriddenscale = Float64(NaN), + overriddenoffset = Float64(NaN), + overriddendstnodata = Float64(NaN), +)::AbstractMDArray + @assert !isnull(mdarray) + return MDArray( + GDAL.gdalmdarraygetunscaled( + mdarray, + overriddenscale, + overriddenoffset, + overriddendstnodata, + ), + mdarray.dataset.value, + ) end -function getvectorlayernames( - group::AbstractGroup, - options::AbstractVector{<:AbstractString} = String[], -)::AbstractVector{<:AbstractString} - return GDAL.gdalgroupgetvectorlayernames(group, options) -end - -function unsafe_openvectorlayer( - group::AbstractGroup, - options, -)::AbstractFeatureLayer - # TODO: Find out how to set `ownedby` and `spatialref`, probably by querying `group` - return FeatureLayer( - GDAL.openvectorlayer(group, options), - ownedby, - spatialref, +function getunscaled( + mdarray::AbstractMDArray, + overriddenscale = Float64(NaN), + overriddenoffset = Float64(NaN), + overriddendstnodata = Float64(NaN), +)::AbstractMDArray + @assert !isnull(mdarray) + return IMDArray( + GDAL.gdalmdarraygetunscaled( + mdarray, + overriddenscale, + overriddenoffset, + overriddendstnodata, + ), + mdarray.dataset.value, ) end -function openvectorlayer(group::AbstractGroup, options)::AbstractFeatureLayer - # TODO: Find out how to set `ownedby` and `spatialref`, probably by querying `group` - return IFeatureLayer( - GDAL.openvectorlayer(group, options), - ownedby, - spatialref, +function unsafe_getmask( + mdarray::AbstractMDArray, + options::OptionList = nothing, +)::AbstractMDArray + @assert !isnull(mdarray) + return MDArray( + GDAL.gdalmdarraygetmask(mdarray, CSLConstListWrapper(options)), + mdarray.dataset.value, ) end -function getdimensions( - group::AbstractGroup, - options::AbstractVector{<:AbstractString} = String[], -)::AbstractVector{<:AbstractDimension} - dimensionscountref = Ref{Csize_t}() - dimensionshptr = - GDAL.gdalgroupgetdimensions(group, dimensionscountref, options) - dimensions = AbstractDimension[ - IDimension(unsafe_load(dimensionshptr, n)) for - n in 1:dimensionscountref[] - ] - GDAL.vsifree(dimensionshptr) - return dimensions +function getmask( + mdarray::AbstractMDArray, + options::OptionList = nothing, +)::AbstractMDArray + @assert !isnull(mdarray) + return IMDArray( + GDAL.gdalmdarraygetmask(mdarray, CSLConstListWrapper(options)), + mdarray.dataset.value, + ) end -function unsafe_creategroup( - group::AbstractGroup, - name::AbstractString, - options::AbstractVector{<:AbstractString} = String[], -)::AbstractGroup - return Group(GDAL.gdalgroupcreategroup(group, name, options)) -end - -function creategroup( - group::AbstractGroup, - name::AbstractString, - options::AbstractVector{<:AbstractString} = String[], -)::AbstractGroup - return IGroup(GDAL.gdalgroupcreategroup(group, name, options)) -end - -function deletegroup(group::AbstractGroup, name::AbstractString, options)::Bool - return GDAL.gdalgroupdeletegroup(group, name, options) -end - -function unsafe_createdimension( - group::AbstractGroup, - name::AbstractString, - type::AbstractString, - direction::AbstractString, - size::Integer, - options::AbstractVector{<:AbstractString} = String[], -)::AbstractDimension - return Dimension( - GDAL.gdalgroupcreatedimension( - group, - name, - type, - direction, - size, - options, +# TODO: Wrap GDAL.GDALRIOResampleAlg +function unsafe_getresampled( + mdarray::AbstractMDArray, + newdims::Union{Nothing,AbstractVector{<:AbstractDimension}}, + resamplealg::GDAL.GDALRIOResampleAlg, + targetsrs::Union{Nothing,AbstractSpatialRef}, + options::OptionList = nothing, +)::AbstractMDArray + @assert !isnull(mdarray) + return MDArray( + GDAL.gdalmdarraygetresampled( + mdarray, + newdims === nothing ? 0 : length(newdims), + newdims === nothing ? C_NULL : DimensionHList(newdims), + resamplealg, + targetsrs == nothing ? C_NULL : targetsrs, + CSLConstListWrapper(options), ), + mdarray.dataset.value, ) end -function createdimension( - group::AbstractGroup, - name::AbstractString, - type::AbstractString, - direction::AbstractString, - size::Integer, - options::AbstractVector{<:AbstractString} = String[], -)::AbstractDimension - return IDimension( - GDAL.gdalgroupcreatedimension( - group, - name, - type, - direction, - size, - options, +function getresampled( + mdarray::AbstractMDArray, + newdims::Union{Nothing,AbstractVector{<:AbstractDimension}}, + resamplealg::GDAL.GDALRIOResampleAlg, + targetsrs::Union{Nothing,AbstractSpatialRef}, + options::OptionList = nothing, +)::AbstractMDArray + @assert !isnull(mdarray) + return IMDArray( + GDAL.gdalmdarraygetresampled( + mdarray, + newdims === nothing ? 0 : length(newdims), + newdims === nothing ? C_NULL : DimensionHList(newdims), + resamplealg, + targetsrs == nothing ? C_NULL : targetsrs, + CSLConstListWrapper(options), ), + mdarray.dataset.value, ) end -function unsafe_createmdarray( - group::AbstractGroup, - name::AbstractString, - dimensions::AbstractVector{<:AbstractDimension}, - datatype::AbstractExtendedDataType, - options::AbstractVector{<:AbstractString} = String[], +function unsafe_getgridded( + mdarray::AbstractMDArray, + gridoptions::AbstractString, + xarray::AbstractMDArray, + yarray::AbstractMDArray, + options::OptionList = nothing, )::AbstractMDArray + @assert !isnull(mdarray) + @assert !isnull(xarray) + @assert !isnull(yarray) return MDArray( - GDAL.gdalgroupcreatemdarray( - group, - name, - length(dimensions), - DimensionHList(dimensions), - datatype, - options, + GDAL.gdalmdarraygetgridded( + mdarray, + gridoptions, + xarray, + yarray, + CSLConstListWrapper(options), ), + mdarray.dataset.value, ) end -function createmdarray( - group::AbstractGroup, - name::AbstractString, - dimensions::AbstractVector{<:AbstractDimension}, - datatype::AbstractExtendedDataType, - options::AbstractVector{<:AbstractString} = String[], +function getgridded( + mdarray::AbstractMDArray, + gridoptions::AbstractString, + xarray::AbstractMDArray, + yarray::AbstractMDArray, + options::OptionList = nothing, )::AbstractMDArray + @assert !isnull(mdarray) + @assert !isnull(xarray) + @assert !isnull(yarray) return IMDArray( - GDAL.gdalgroupcreatemdarray( - group, - name, - length(dimensions), - DimensionHList(dimensions), - datatype, - options, + GDAL.gdalmdarraygetgridded( + mdarray, + gridoptions, + xarray, + yarray, + CSLConstListWrapper(options), ), + mdarray.dataset.value, ) end -function deletemdarray( - group::AbstractGroup, - name::AbstractString, - options::AbstractVector{<:AbstractString} = String[], -)::Bool - return GDAL.gdalgroupdeletemdarray(group, name, options) +function unsafe_asclassicdataset( + mdarray::AbstractMDArray, + xdim::Integer, + ydim::Integer, + rootgroup::Union{Nothing,AbstractGroup} = nothing, + options::OptionList = nothing, +)::AbstractDataset + @assert !isnull(mdarray) + @assert rootgroup === nothing || !isnull(rootgroup) + return Dataset( + GDAL.gdalmdarrayasclassicdataset( + mdarray, + xdim, + ydim, + rootgroup === nothing ? C_NULL : rootgroup, + CSLConstListWrapper(options), + ), + ) end -# gettotalcopycost -# copyfrom - -function getstructuralinfo( - group::AbstractGroup, -)::AbstractVector{<:AbstractString} - return GDAL.gdalgroupgetstructuralinfo(group) +function asclassicdataset( + mdarray::AbstractMDArray, + xdim::Integer, + ydim::Integer, + rootgroup::Union{Nothing,AbstractGroup} = nothing, + options::OptionList = nothing, +)::AbstractDataset + @assert !isnull(mdarray) + @assert rootgroup === nothing || !isnull(rootgroup) + return IDataset( + GDAL.gdalmdarrayasclassicdataset( + mdarray, + xdim, + ydim, + rootgroup === nothing ? C_NULL : rootgroup, + CSLConstListWrapper(options), + ), + ) end -function unsafe_openmdarrayfromfullname( - group::AbstractGroup, - fullname::AbstractString, - options::AbstractVector{<:AbstractString} = String[], -)::AbstractMDArray - return MDArray( - GDAL.gdalgroupopenmdarrayfromfullname(group, fullname, options), +function unsafe_asmdarray(rasterband::AbstractRasterBand)::AbstractMDArray + @assert !isnull(rasterband) + # TODO: Find dataset + return MDArray(GADL.gdalrasterbandasmdarray(rasterband)) +end + +function asmdarray(rasterband::AbstractRasterBand)::AbstractMDArray + @assert !isnull(rasterband) + # TODO: Find dataset + return IMDArray(GADL.gdalrasterbandasmdarray(rasterband)) +end + +# TODO: Wrap GDAL.CPLErr +# TODO: Allow a progress function +function getstatistics( + mdarray::AbstractMDArray, + approxok::Bool, + force::Bool, +)::Tuple{GDAL.CPLErr,Float64,Float64,Float64,Float64,Int64} + @assert !isnull(mdarray) + dataset = C_NULL # apparently unused + min = Ref{Float64}() + max = Ref{Float64}() + mean = Ref{Float64}() + stddev = Ref{Float64}() + validcount = Ref{UInt64}() + err = GDAL.gdalmdarraygetstatistics( + mdarray, + dataset, + approxok, + force, + min, + max, + mean, + stddev, + validcount, + C_NULL, + C_NULL, ) + return err, min[], max[], mean[], stddev[], Int64(validcount[]) +end + +# TODO: Allow a progress function +function computestatistics( + mdarray::AbstractMDArray, + approxok::Bool, + options::OptionList = nothing, +)::Tuple{Bool,Float64,Float64,Float64,Float64,Int64} + @assert !isnull(mdarray) + dataset = C_NULL # apparently unused + min = Ref{Float64}() + max = Ref{Float64}() + mean = Ref{Float64}() + stddev = Ref{Float64}() + validcount = Ref{UInt64}() + success = GDAL.gdalmdarraycomputestatisticsex( + mdarray, + dataset, + approxok, + min, + max, + mean, + stddev, + validcount, + C_NULL, + C_NULL, + CSLConstListWrapper(options), + ) + return succeess, min[], max[], mean[], stddev[], Int64(validcount[]) end -function openmdarrayfromfullname( - group::AbstractGroup, - fullname::AbstractString, - options::AbstractVector{<:AbstractString} = String[], -)::AbstractMDArray - return IMDArray( - GDAL.gdalgroupopenmdarrayfromfullname(group, fullname, options), - ) +function clearstatistics(mdarray::AbstractMDArray)::Nothing + @assert !isnull(mdarray) + return GDAL.gdalmdarrayclearstatistics(mdarray) end -function unsafe_resolvemdarray( - group::AbstractGroup, - name::AbstractString, - startingpath::AbstractString, - options::AbstractVector{<:AbstractString} = String[], -)::AbstractMDArray - return MDArray( - GDAL.gdalgroupresolvemdarray(group, name, startingpath, options), +function getcoordinatevariables( + mdarray::AbstractMDArray, +)::AbstractVector{<:AbstractMDArray} + @assert !isnull(mdarray) + count = Ref{Csize_t}() + coordinatevariablesptr = + GDAL.gdalmdarraygetcoordinatevariables(mdarray, count) + dataset = mdarray.dataset.value + coordinatevariables = AbstractMDArray[ + IMDArray(unsafe_load(coordinatevariablesptr, n), dataset) for + n in 1:count[] + ] + GDAL.vsifree(coordinatevariablesptr) + return coordinatevariables +end + +function adviseread( + mdarray::AbstractMDArray, + arraystartidx::Union{Nothing,AbstractVector{<:Integer}}, + count::Union{Nothing,AbstractVector{<:Integer}}, + options::OptionList = nothing, +)::Bool + @assert !isnull(mdarray) + return GDAL.gdalmdarrayadviseread( + mdarray, + arraystartidx === nothing ? C_NULL : arraystartidx, + count === nothing ? C_NULL : count, + CSLConstListWrapper(options), ) end -function resolvemdarray( - group::AbstractGroup, - name::AbstractString, - startingpath::AbstractString, - options::AbstractVector{<:AbstractString} = String[], -)::AbstractMDArray - return IMDArray( - GDAL.gdalgroupresolvemdarray(group, name, startingpath, options), +function isregularlyspaced( + mdarray::AbstractMDArray, +)::Union{Nothing,Tuple{Float64,Float64}} + @assert !isnull(mdarray) + start = Ref{Float64}() + increment = Ref{Float64}() + res = GDAL.gdalmdarrayisregularlyspaced(mdarray, start, increment) + !res[] && return nothing + return start[], increment[] +end + +function guessgeotransform( + mdarray::AbstractMDArray, + dimx::Integer, + dimy::Integer, + pixelispoint::Bool, +)::Union{Nothing,AbstractVector{Float64}} + @assert !isnull(mdarray) + geotransform = Vector{Float64}(undef, 6) + res = GDAL.gdalmdarrayguessgeotransform( + mdarray, + dimx, + dimy, + pixelispoint, + geotransform, ) + !res && return nothing + return geotransform end -function unsafe_opengroupfromfullname( - group::AbstractGroup, - fullname::AbstractString, - options::AbstractVector{<:AbstractString} = String[], -)::AbstractGroup - return Group(GDAL.gdalgroupopengroupfromfullname(group, fullname, options)) +function cache(mdarray::AbstractMDArray, options::OptionList = nothing)::Bool + @assert !isnull(mdarray) + return GDAL.gdalmdarraycache(mdarray, CSLConstListWrapper(options)) end -function opengroupfromfullname( - group::AbstractGroup, - fullname::AbstractString, - options::AbstractVector{<:AbstractString} = String[], -)::AbstractGroup - return IGroup(GDAL.gdalgroupopengroupfromfullname(group, fullname, options)) +function getrootgroup(mdarray::AbstractMDArray)::AbstractGroup + @assert !isnull(mdarray) + return Group(GDAL.gdalmdarraygetrootgroup(mdarray), mdarray.dataset.value) end -# function unsafe_opendimensionfromfullname( -# group::AbstractGroup, -# fullname::AbstractString, -# options::AbstractVector{<:AbstractString} = String[], -# )::AbstractDimension -# return Dimension( -# GDAL.gdalgroupopendimensionfromfullname(group, fullname, options), -# ) -# end -# -# function opendimensionfromfullname( -# group::AbstractGroup, -# fullname::AbstractString, -# options::AbstractVector{<:AbstractString} = String[], -# )::AbstractDimension -# return IDimension( -# GDAL.gdalgroupopendimensionfromfullname(group, fullname, options), -# ) -# end +################################################################################ -# clearstatistics +const AbstractAttributeOrMDArray = Union{AbstractAttribute,AbstractMDArray} -function rename(group::AbstractGroup, newname::AbstractString)::Bool - return GDAL.gdalgrouprename(group, newname) +function getname(mdarray::AbstractAttributeOrMDArray)::AbstractString + @assert !isnull(mdarray) + return GDAL.gdalmdarraygetname(mdarray) end -function unsafe_subsetdimensionfromselection( - group::AbstractGroup, - selection::AbstractString, - options::AbstractVector{<:AbstractString} = String[], -)::AbstractGroup - return Group( - GDAL.gdalgroupsubsetdimensionfromselection(group, selection, options), - ) +function getfullname(mdarray::AbstractAttributeOrMDArray)::AbstractString + @assert !isnull(mdarray) + return GDAL.gdalmdarraygetfullname(mdarray) end -function subsetdimensionfromselection( - group::AbstractGroup, - selection::AbstractString, - options::AbstractVector{<:AbstractString} = String[], -)::AbstractGroup - return IGroup( - GDAL.gdalgroupsubsetdimensionfromselection(group, selection, options), - ) +function gettotalelementscount(mdarray::AbstractAttributeOrMDArray)::Int64 + @assert !isnull(mdarray) + return Int64(GDAL.gdalmdarraygettotalelementscount(mdarray)) end -################################################################################ +function Base.length(mdarray::AbstractAttributeOrMDArray) + @assert !isnull(mdarray) + return Int(gettotalelementscount(mdarray)) +end + +function getdimensioncount(mdarray::AbstractAttributeOrMDArray)::Int + @assert !isnull(mdarray) + return Int(GDAL.gdalmdarraygetdimensioncount(mdarray)) +end -# GDALExtendedDataType +Base.ndims(mdarray::AbstractAttributeOrMDArray)::Int = + getdimensioncount(mdarray) -function unsafe_extendeddatatypecreate( - ::Type{T}, -)::AbstractExtendedDataType where {T} - type = convert(GDALDataType, T) - return ExtendedDataType(GDAL.gdalextendeddatatypecreate(type)) +function getdimensions( + mdarray::AbstractAttributeOrMDArray, +)::AbstractVector{<:AbstractDimension} + @assert !isnull(mdarray) + dimensionscountref = Ref{Csize_t}() + dimensionshptr = GDAL.gdalmdarraygetdimensions(mdarray, dimensionscountref) + dataset = mdarray.dataset.value + dimensions = AbstractDimension[ + IDimension(unsafe_load(dimensionshptr, n), dataset) for + n in 1:dimensionscountref[] + ] + GDAL.vsifree(dimensionshptr) + return dimensions end -function extendeddatatypecreate(::Type{T})::AbstractExtendedDataType where {T} - type = convert(GDALDataType, T) - return IExtendedDataType(GDAL.gdalextendeddatatypecreate(type)) +function unsafe_getdimensions( + mdarray::AbstractAttributeOrMDArray, +)::AbstractVector{<:AbstractDimension} + @assert !isnull(mdarray) + dimensionscountref = Ref{Csize_t}() + dimensionshptr = GDAL.gdalmdarraygetdimensions(mdarray, dimensionscountref) + dataset = mdarray.dataset.value + dimensions = AbstractDimension[ + Dimension(unsafe_load(dimensionshptr, n), dataset) for + n in 1:dimensionscountref[] + ] + GDAL.vsifree(dimensionshptr) + return dimensions end -################################################################################ +function unsafe_getdatatype( + mdarray::AbstractAttributeOrMDArray, +)::AbstractExtendedDataType + @assert !isnull(mdarray) + return ExtendedDataType(GDAL.gdalmdarraygetdatatype(mdarray)) +end -# GDALMDArray +function getdatatype( + mdarray::AbstractAttributeOrMDArray, +)::AbstractExtendedDataType + @assert !isnull(mdarray) + return IExtendedDataType(GDAL.gdalmdarraygetdatatype(mdarray)) +end -function getfilename(mdarray::AbstractMDArray)::AbstractString - return GDAL.gdalmdarraygetfilename(mdarray) +function getblocksize( + mdarray::AbstractAttributeOrMDArray, + options::OptionList = nothing, +)::AbstractVector{Int64} + @assert !isnull(mdarray) + count = Ref{Csize_t}() + blocksizeptr = GDAL.gdalmdarraygetblocksize( + mdarray, + count, + CSLConstListWrapper(options), + ) + blocksize = Int64[unsafe_load(blocksizeptr, n) for n in 1:count[]] + GDAL.vsifree(blocksizeptr) + return blocksize +end + +function getprocessingchunksize( + mdarray::AbstractAttributeOrMDArray, + maxchunkmemory::Integer, +)::AbstractVector{Int64} + @assert !isnull(mdarray) + count = Ref{Csize_t}() + chunksizeptr = + GDAL.gdalmdarraygetprocessingchunksize(mdarray, count, maxchunkmemory) + chunksize = Int64[unsafe_load(chunksizeptr, n) for n in 1:count[]] + GDAL.vsifree(chunksizeptr) + return chunksize +end + +# processperchunk + +const IndexLike{D} = + Union{AbstractVector{<:Integer},CartesianIndex{D},NTuple{D,<:Integer}} +const RangeLike{D} = Union{ + AbstractVector{<:AbstractRange{<:Integer}}, + NTuple{D,<:AbstractRange{<:Integer}}, +} + +function read!( + mdarray::AbstractAttributeOrMDArray, + arraystartidx::IndexLike{D}, + count::IndexLike{D}, + arraystep::Union{Nothing,IndexLike{D}}, + buffer::StridedArray{T,D}, +)::Bool where {T,D} + @assert !isnull(mdarray) + @assert length(arraystartidx) == D + @assert length(count) == D + @assert arraystep === nothing ? true : length(arraystep) == D + gdal_arraystartidx = UInt64[arraystartidx[n] - 1 for n in D:-1:1] + gdal_count = Csize_t[count[n] for n in D:-1:1] + gdal_arraystep = + arraystep === nothing ? nothing : Int64[arraystep[n] for n in D:-1:1] + gdal_bufferstride = Cptrdiff_t[stride(buffer, n) for n in D:-1:1] + return extendeddatatypecreate(T) do bufferdatatype + return GDAL.gdalmdarrayread( + mdarray, + gdal_arraystartidx, + gdal_count, + gdal_arraystep, + gdal_bufferstride, + bufferdatatype, + buffer, + buffer, + sizeof(buffer), + ) + end +end + +function read!( + mdarray::AbstractAttributeOrMDArray, + region::RangeLike{D}, + buffer::StridedArray{T,D}, +)::Bool where {T,D} + @assert length(region) == D + arraystartidx = first.(region) + count = length.(region) + arraystep = step.(region) + return read!(mdarray, arraystartidx, count, arraystep, buffer) +end + +function read!( + mdarray::AbstractAttributeOrMDArray, + indices::CartesianIndices{D}, + arraystep::Union{Nothing,IndexLike{D}}, + buffer::StridedArray{T,D}, +)::Bool where {T,D} + @assert length(region) == D + arraystartidx = first.(indices) + count = length.(indices) + return read!(mdarray, arraystartidx, count, arraystep, buffer) +end + +function read!( + mdarray::AbstractAttributeOrMDArray, + indices::CartesianIndices{D}, + buffer::StridedArray{T,D}, +)::Bool where {T,D} + return read!(mdarray, indices, nothing, buffer) +end + +function read!( + mdarray::AbstractAttributeOrMDArray, + buffer::StridedArray{T,D}, +)::Bool where {T,D} + return read!(mdarray, axes(buffer), buffer) +end + +function write( + mdarray::AbstractAttributeOrMDArray, + arraystartidx::IndexLike{D}, + count::IndexLike{D}, + arraystep::Union{Nothing,IndexLike{D}}, + buffer::StridedArray{T,D}, +)::Bool where {T,D} + @assert !isnull(mdarray) + @assert length(arraystartidx) == D + @assert length(count) == D + @assert arraystep === nothing ? true : length(arraystep) == D + gdal_arraystartidx = UInt64[arraystartidx[n] - 1 for n in D:-1:1] + gdal_count = Csize_t[count[n] for n in D:-1:1] + gdal_arraystep = + arraystep === nothing ? nothing : Int64[arraystep[n] for n in D:-1:1] + gdal_bufferstride = Cptrdiff_t[stride(buffer, n) for n in D:-1:1] + return extendeddatatypecreate(T) do bufferdatatype + return GDAL.gdalmdarraywrite( + mdarray, + gdal_arraystartidx, + gdal_count, + gdal_arraystep, + gdal_bufferstride, + bufferdatatype, + buffer, + buffer, + sizeof(buffer), + ) + end +end + +function write( + mdarray::AbstractAttributeOrMDArray, + region::RangeLike{D}, + buffer::StridedArray{T,D}, +)::Bool where {T,D} + @assert length(region) == D + arraystartidx = first.(region) + count = length.(region) + arraystep = step.(region) + return write(mdarray, arraystartidx, count, arraystep, buffer) +end + +function write( + mdarray::AbstractAttributeOrMDArray, + indices::CartesianIndices{D}, + arraystep::Union{Nothing,IndexLike{D}}, + buffer::StridedArray{T,D}, +)::Bool where {T,D} + @assert length(region) == D + arraystartidx = first.(indices) + count = length.(indices) + return write(mdarray, arraystartidx, count, arraystep, buffer) +end + +function write( + mdarray::AbstractAttributeOrMDArray, + indices::CartesianIndices{D}, + buffer::StridedArray{T,D}, +)::Bool where {T,D} + return write(mdarray, indices, nothing, buffer) +end + +function write( + mdarray::AbstractAttributeOrMDArray, + buffer::StridedArray{T,D}, +)::Bool where {T,D} + return write(mdarray, axes(buffer), buffer) +end + +function rename!(mdarray::AbstractMDArray, newname::AbstractString)::Bool + @assert !isnull(mdarray) + return GDAL.gdalmdarrayrename(mdarray, newname) end diff --git a/src/mdarray/todo.jl b/src/mdarray/todo.jl new file mode 100644 index 00000000..cd4e6301 --- /dev/null +++ b/src/mdarray/todo.jl @@ -0,0 +1,13 @@ +# GDALExtendedDataType + +function unsafe_extendeddatatypecreate( + ::Type{T}, +)::AbstractExtendedDataType where {T} + type = convert(GDALDataType, T) + return ExtendedDataType(GDAL.gdalextendeddatatypecreate(type)) +end + +function extendeddatatypecreate(::Type{T})::AbstractExtendedDataType where {T} + type = convert(GDALDataType, T) + return IExtendedDataType(GDAL.gdalextendeddatatypecreate(type)) +end diff --git a/src/mdarray/types.jl b/src/mdarray/types.jl new file mode 100644 index 00000000..4e04a8ff --- /dev/null +++ b/src/mdarray/types.jl @@ -0,0 +1,282 @@ +abstract type AbstractExtendedDataType end +# needs to have a `ptr::GDALExtendedDataTypeH` attribute + +abstract type AbstractEDTComponent end +# needs to have a `ptr::GDALEDTComponentH` attribute + +abstract type AbstractGroup end +# needs to have a `ptr::GDAL.GDALGroupH` attribute + +# TODO: Add `<: AbstractDiskArray` +# TODO: Put `{T,D}` into the type signature? +abstract type AbstractMDArray end +# needs to have a `ptr::GDAL.GDALMDArrayH` attribute + +abstract type AbstractAttribute end +# needs to have a `ptr::GDAL.GDALAttributeH` attribute + +abstract type AbstractDimension end +# needs to have a `ptr::GDAL.GDALDimensionH` attribute + +################################################################################ + +mutable struct ExtendedDataType <: AbstractExtendedDataType + ptr::GDAL.GDALExtendedDataTypeH + + ExtendedDataType(ptr::GDAL.GDALExtendedDataTypeH) = new(ptr) +end + +mutable struct IExtendedDataType <: AbstractExtendedDataType + ptr::GDAL.GDALExtendedDataTypeH + + function IExtendedDataType(ptr::GDAL.GDALExtendedDataTypeH) + extendeddatatype = new(ptr) + ptr != C_NULL && finalizer(destroy, extendeddatatype) + return extendeddatatype + end +end + +mutable struct EDTComponent <: AbstractEDTComponent + ptr::GDAL.GDALEDTComponentH + + EDTComponent(ptr::GDAL.GDALEDTComponentH) = new(ptr) +end + +mutable struct IEDTComponent <: AbstractEDTComponent + ptr::GDAL.GDALEDTComponentH + dataset::WeakRef # AbstractDataset + + function IEDTComponent(ptr::GDAL.GDALEDTComponentH) + edtcomponent = new(ptr) + ptr != C_NULL && finalizer(destroy, edtcomponent) + return edtcomponent + end +end + +mutable struct Group <: AbstractGroup + ptr::GDAL.GDALGroupH + dataset::WeakRef # AbstractDataset + + function Group(ptr::GDAL.GDALGroupH, dataset::AbstractDataset) + group = new(ptr, WeakRef(dataset)) + add_child!(dataset, group) + return group + end +end + +mutable struct IGroup <: AbstractGroup + ptr::GDAL.GDALGroupH + dataset::WeakRef # AbstractDataset + + function IGroup(ptr::GDAL.GDALGroupH, dataset::AbstractDataset) + group = new(ptr, WeakRef(dataset)) + add_child!(dataset, group) + ptr != C_NULL && finalizer(destroy, group) + return group + end +end + +mutable struct MDArray <: AbstractMDArray + ptr::GDAL.GDALMDArrayH + dataset::WeakRef # AbstractDataset + + function MDArray(ptr::GDAL.GDALMDArrayH, dataset::AbstractDataset) + mdarray = new(ptr, WeakRef(dataset)) + add_child!(dataset, mdarray) + return mdarray + end +end + +mutable struct IMDArray <: AbstractMDArray + ptr::GDAL.GDALMDArrayH + dataset::WeakRef # AbstractDataset + + function IMDArray(ptr::GDAL.GDALMDArrayH, dataset::AbstractDataset) + mdarray = new(ptr, WeakRef(dataset)) + add_child!(dataset, mdarray) + ptr != C_NULL && finalizer(destroy, mdarray) + return mdarray + end +end + +mutable struct Attribute <: AbstractAttribute + ptr::GDAL.GDALAttributeH + dataset::WeakRef # AbstractDataset + + function Attribute(ptr::GDAL.GDALAttributeH, dataset::AbstractDataset) + attribute = new(ptr, WeakRef(dataset)) + add_child!(dataset, attribute) + return attribute + end +end + +mutable struct IAttribute <: AbstractAttribute + ptr::GDAL.GDALAttributeH + dataset::WeakRef # AbstractDataset + + function IAttribute(ptr::GDAL.GDALAttributeH, dataset::AbstractDataset) + attribute = new(ptr, WeakRef(dataset)) + add_child!(dataset, attribute) + ptr != C_NULL && finalizer(destroy, attribute) + return attribute + end +end + +mutable struct Dimension <: AbstractDimension + ptr::GDAL.GDALDimensionH + dataset::WeakRef # AbstractDataset + + function Dimension(ptr::GDAL.GDALDimensionH, dataset::AbstractDataset) + dimension = new(ptr, WeakRef(dataset)) + add_child!(dataset, dimension) + return dimension + end +end + +mutable struct IDimension <: AbstractDimension + ptr::GDAL.GDALDimensionH + dataset::WeakRef # AbstractDataset + + function IDimension(ptr::GDAL.GDALDimensionH, dataset::AbstractDataset) + dimension = new(ptr, WeakRef(dataset)) + add_child!(dataset, dimension) + ptr != C_NULL && finalizer(destroy, dimension) + return dimension + end +end + +################################################################################ + +isnull(x::AbstractAttribute) = x.ptr == C_NULL +isnull(x::AbstractDataset) = x.ptr == C_NULL +isnull(x::AbstractDimension) = x.ptr == C_NULL +isnull(x::AbstractEDTComponent) = x.ptr == C_NULL +isnull(x::AbstractExtendedDataType) = x.ptr == C_NULL +isnull(x::AbstractFeature) = x.ptr == C_NULL +isnull(x::AbstractFeatureDefn) = x.ptr == C_NULL +isnull(x::AbstractFeatureLayer) = x.ptr == C_NULL +isnull(x::AbstractFieldDefn) = x.ptr == C_NULL +isnull(x::AbstractGeomFieldDefn) = x.ptr == C_NULL +isnull(x::AbstractGeometry) = x.ptr == C_NULL +isnull(x::AbstractGroup) = x.ptr == C_NULL +isnull(x::AbstractMDArray) = x.ptr == C_NULL +isnull(x::AbstractRasterBand) = x.ptr == C_NULL +isnull(x::AbstractSpatialRef) = x.ptr == C_NULL +isnull(x::ColorTable) = x.ptr == C_NULL +isnull(x::CoordTransform) = x.ptr == C_NULL +isnull(x::Driver) = x.ptr == C_NULL +isnull(x::Field) = x.ptr == C_NULL +isnull(x::RasterAttrTable) = x.ptr == C_NULL +isnull(x::StyleManager) = x.ptr == C_NULL +isnull(x::StyleTable) = x.ptr == C_NULL +isnull(x::StyleTool) = x.ptr == C_NULL + +################################################################################ + +Base.unsafe_convert(::Type{Ptr{Cvoid}}, x::AbstractExtendedDataType) = x.ptr +Base.unsafe_convert(::Type{Ptr{Cvoid}}, x::AbstractEDTComponent) = x.ptr +Base.unsafe_convert(::Type{Ptr{Cvoid}}, x::AbstractGroup) = x.ptr +Base.unsafe_convert(::Type{Ptr{Cvoid}}, x::AbstractMDArray) = x.ptr +Base.unsafe_convert(::Type{Ptr{Cvoid}}, x::AbstractAttribute) = x.ptr +Base.unsafe_convert(::Type{Ptr{Cvoid}}, x::AbstractDimension) = x.ptr + +################################################################################ + +function destroy(datatype::AbstractExtendedDataType)::Nothing + datatype.ptr == C_NULL && return nothing + GDAL.gdalextendeddatatyperelease(datatype) + datatype.ptr = C_NULL + return nothing +end + +function destroy(edtcomponent::AbstractEDTComponent)::Nothing + edtcomponent.ptr == C_NULL && return nothing + GDAL.gdaledtcomponentrelease(edtcomponent) + edtcomponent.ptr = C_NULL + return nothing +end + +function destroy(group::AbstractGroup)::Nothing + group.ptr == C_NULL && return nothing + GDAL.gdalgrouprelease(group) + group.ptr = C_NULL + return nothing +end + +function destroy(mdarray::AbstractMDArray)::Nothing + mdarray.ptr == C_NULL && return nothing + GDAL.gdalmdarrayrelease(mdarray) + mdarray.ptr = C_NULL + return nothing +end + +function destroy(attribute::AbstractAttribute)::Nothing + attribute.ptr == C_NULL && return nothing + GDAL.gdalattributerelease(attribute) + attribute.ptr = C_NULL + return nothing +end + +function destroy(dimension::AbstractDimension)::Nothing + dimension.ptr == C_NULL && return nothing + GDAL.gdaldimensionrelease(dimension) + dimension.ptr = C_NULL + return nothing +end + +destroy(dimensions::AbstractVector{<:AbstractDimension}) = destroy.(dimensions) + +################################################################################ + +# Helpers + +const OptionList = Union{Nothing,AbstractVector{<:AbstractString}} + +# TODO: Move to GDAL.jl, redefining `CSLConstList` +struct CSLConstListWrapper + # Hold on to the original arguments to prevent GC from freeing + # them while they are being used in a ccall + cstrings::Union{Nothing,Vector{Cstring}} + strings::Any + + function CSLConstListWrapper(strings::Nothing) + cstrings = nothing + return new(cstrings, strings) + end + function CSLConstListWrapper( + strings::AbstractVector{<:Union{String,SubString{String}}}, + ) + cstrings = Cstring[[pointer(str) for str in strings]; C_NULL] + return new(cstrings, strings) + end +end +function CSLConstListWrapper(strings::AbstractVector{<:AbstractString}) + return CSLConstListWrapper(String.(strings)) +end + +function Base.cconvert(::Type{GDAL.CSLConstList}, wrapper::CSLConstListWrapper) + wrapper.cstrings === nothing && + return Base.cconvert(GDAL.CSLConstList, C_NULL) + return Base.cconvert(GDAL.CSLConstList, wrapper.cstrings) +end + +struct DimensionHList + # Hold on to the original arguments to prevent GC from freeing + # them while they are being used in a ccall + dimensionhs::Vector{GDAL.GDALDimensionH} + dimensions::AbstractVector{<:AbstractDimension} + + function DimensionHList(dimensions::AbstractVector{<:AbstractDimension}) + dimensionhs = GDAL.GDALDimensionH[ + Base.unsafe_convert(Ptr{Cvoid}, dim) for dim in dimensions + ] + return new(dimensionhs, dimensions) + end +end + +function Base.cconvert( + ::Type{Ptr{GDAL.GDALDimensionH}}, + dimensionhlist::DimensionHList, +) + return Base.cconvert(Ptr{Cvoid}, dimensionhlist.dimensionhs) +end diff --git a/src/types.jl b/src/types.jl index 38e85909..34eb2ef4 100644 --- a/src/types.jl +++ b/src/types.jl @@ -35,22 +35,45 @@ mutable struct CoordTransform ptr::GDAL.OGRCoordinateTransformationH end +# In the multidim API, underlying files are closed only when all +# objects potentially pointing to them (groups, arrays, attributes, +# dimensions) have been released. Their lifetime is not connected with +# the one of the GDALDataset. +# +# To handle this in Julia, each multidim dataset can hold a list of +# its children. This allows us to "hard close" a dataset when using +# the interactive API. Being able to close a dataset at a particular +# time is important when writing. +# +# Each child must have a `destroy` function. mutable struct Dataset <: AbstractDataset ptr::GDAL.GDALDatasetH + children::Union{Nothing,Vector{Any}} - Dataset(ptr::GDAL.GDALDatasetH = C_NULL) = new(ptr) + function Dataset(ptr::GDAL.GDALDatasetH = C_NULL; hard_close::Bool = false) + return new(ptr, hard_close ? [] : nothing) + end end mutable struct IDataset <: AbstractDataset ptr::GDAL.GDALDatasetH + children::Union{Nothing,Vector{Any}} - function IDataset(ptr::GDAL.GDALDatasetH = C_NULL) - dataset = new(ptr) + function IDataset(ptr::GDAL.GDALDatasetH = C_NULL; hard_close::Bool = false) + dataset = new(ptr, hard_close ? [] : nothing) finalizer(destroy, dataset) return dataset end end +function add_child!(dataset::AbstractDataset, obj::Any)::Nothing + isnull(obj) && return nothing + @assert !isnull(dataset) + dataset.children === nothing && return nothing + push!(dataset.children, obj) + return nothing +end + mutable struct Driver ptr::GDAL.GDALDriverH end @@ -615,13 +638,14 @@ end wkbNDR::GDAL.wkbNDR, ) -import Base.| - for T in (GDALOpenFlag, FieldValidation) eval(quote - |(x::$T, y::UInt8)::UInt8 = UInt8(x) | y - |(x::UInt8, y::$T)::UInt8 = x | UInt8(y) - |(x::$T, y::$T)::UInt8 = UInt8(x) | UInt8(y) + Base.:&(x::$T, y::UInt32)::UInt32 = UInt32(x) & y + Base.:&(x::UInt32, y::$T)::UInt32 = x & UInt32(y) + Base.:&(x::$T, y::$T)::UInt32 = UInt32(x) & UInt32(y) + Base.:|(x::$T, y::UInt32)::UInt32 = UInt32(x) | y + Base.:|(x::UInt32, y::$T)::UInt32 = x | UInt32(y) + Base.:|(x::$T, y::$T)::UInt32 = UInt32(x) | UInt32(y) end) end diff --git a/test/gdal.jl b/test/gdal.jl new file mode 100644 index 00000000..c2b67966 --- /dev/null +++ b/test/gdal.jl @@ -0,0 +1,594 @@ +using Test +import ArchGDAL as AG + +using GDAL + +# using GDAL_jll +# `$(GDAL_jll.gdalinfo_path()) --format Zarr` |> run +# `$(GDAL_jll.gdalmdiminfo_exe()) data.zarr` |> run + +function check_last_error() + error = GDAL.cplgetlasterrormsg() + isempty(error) && return nothing + println(error) + throw(ErrorException(error)) + return nothing +end + +function make_data() + ndishes = 64 + npolarizations = 2 + #TODO nfrequencies = 384 + #TODO ntimes = 8192 + #TODO data = Float32[ + #TODO 1000 * t + 100 * f + 10 * p + 1 * d for d in 0:(ndishes - 1), p in 0:(npolarizations - 1), f in 0:(nfrequencies - 1), + #TODO t in 0:(ntimes - 1) + #TODO ] + data = Float32[ + 10 * p + 1 * d for d in 0:(ndishes-1), p in 0:(npolarizations-1) + ] + return data +end + +function write_attribute( + mdarrayh::GDAL.GDALMDArrayH, + name::AbstractString, + value::AbstractString, +) + datatypeh = GDAL.gdalextendeddatatypecreatestring(length(value)) + check_last_error() + @assert datatypeh != C_NULL + attributeh = GDAL.gdalmdarraycreateattribute( + mdarrayh, + name, + 0, + C_NULL, + datatypeh, + C_NULL, + ) + check_last_error() + @assert attributeh != C_NULL + noerr = GDAL.gdalattributewritestring(attributeh, value) + check_last_error() + @assert noerr == true + GDAL.gdalattributerelease(attributeh) + check_last_error() + GDAL.gdalextendeddatatyperelease(datatypeh) + check_last_error() + return nothing +end + +function write_attribute( + mdarrayh::GDAL.GDALMDArrayH, + name::AbstractString, + value::Integer, +) + datatypeh = GDAL.gdalextendeddatatypecreate(GDAL.GDT_Int64) + check_last_error() + @assert datatypeh != C_NULL + attributeh = GDAL.gdalmdarraycreateattribute( + mdarrayh, + name, + 0, + C_NULL, + datatypeh, + C_NULL, + ) + check_last_error() + @assert attributeh != C_NULL + noerr = + GDAL.gdalattributewriteraw(attributeh, Ref(Int64(value)), sizeof(Int64)) + check_last_error() + @assert noerr == true + GDAL.gdalattributerelease(attributeh) + check_last_error() + GDAL.gdalextendeddatatyperelease(datatypeh) + check_last_error() + return nothing +end + +function write_attribute( + mdarrayh::GDAL.GDALMDArrayH, + name::AbstractString, + value::Real, +) + datatypeh = GDAL.gdalextendeddatatypecreate(GDAL.GDT_Float64) + check_last_error() + @assert datatypeh != C_NULL + attributeh = GDAL.gdalmdarraycreateattribute( + mdarrayh, + name, + 0, + C_NULL, + datatypeh, + C_NULL, + ) + check_last_error() + @assert attributeh != C_NULL + noerr = GDAL.gdalattributewriteraw( + attributeh, + Ref(Float64(value)), + sizeof(Float64), + ) + check_last_error() + @assert noerr == true + GDAL.gdalattributerelease(attributeh) + check_last_error() + GDAL.gdalextendeddatatyperelease(datatypeh) + check_last_error() + return nothing +end + +function write_attribute( + mdarrayh::GDAL.GDALMDArrayH, + name::AbstractString, + values::AbstractArray{<:Integer}, +) + datatypeh = GDAL.gdalextendeddatatypecreate(GDAL.GDT_Int64) + check_last_error() + @assert datatypeh != C_NULL + rank = ndims(values) + sizes = collect(GDAL.GUInt64.(reverse(size(values)))) + attributeh = GDAL.gdalmdarraycreateattribute( + mdarrayh, + name, + rank, + sizes, + datatypeh, + C_NULL, + ) + check_last_error() + @assert attributeh != C_NULL + noerr = GDAL.gdalattributewriteraw( + attributeh, + Int64.(values), + sizeof(Int64) * length(values), + ) + check_last_error() + @assert noerr == true + GDAL.gdalattributerelease(attributeh) + check_last_error() + GDAL.gdalextendeddatatyperelease(datatypeh) + check_last_error() + return nothing +end + +function write_attribute( + mdarrayh::GDAL.GDALMDArrayH, + name::AbstractString, + values::AbstractArray{<:Real}, +) + datatypeh = GDAL.gdalextendeddatatypecreate(GDAL.GDT_Float64) + check_last_error() + @assert datatypeh != C_NULL + rank = ndims(values) + sizes = collect(GDAL.GUInt64.(reverse(size(values)))) + attributeh = GDAL.gdalmdarraycreateattribute( + mdarrayh, + name, + rank, + sizes, + datatypeh, + C_NULL, + ) + check_last_error() + @assert attributeh != C_NULL + noerr = GDAL.gdalattributewriteraw( + attributeh, + Float64.(values), + sizeof(Float64) * length(values), + ) + check_last_error() + @assert noerr == true + GDAL.gdalattributerelease(attributeh) + check_last_error() + GDAL.gdalextendeddatatyperelease(datatypeh) + check_last_error() + return nothing +end + +function read_attribute(mdarrayh::GDAL.GDALMDArrayH, name::AbstractString) + attributeh = GDAL.gdalmdarraygetattribute(mdarrayh, name) + check_last_error() + @assert attributeh != C_NULL + datatypeh = GDAL.gdalattributegetdatatype(attributeh) + check_last_error() + dimensioncount = GDAL.gdalattributegetdimensioncount(attributeh) + check_last_error() + @assert dimensioncount in (0, 1) + class = GDAL.gdalextendeddatatypegetclass(datatypeh) + check_last_error() + if class === GDAL.GEDTC_STRING + @assert dimensioncount == 0 + value = GDAL.gdalattributereadasstring(attributeh) + value::AbstractString + elseif class === GDAL.GEDTC_NUMERIC + attributesizeref = Ref(~Csize_t(0)) + valueptr = GDAL.gdalattributereadasraw(attributeh, attributesizeref) + numericdatatype = GDAL.gdalextendeddatatypegetnumericdatatype(datatypeh) + check_last_error() + if numericdatatype === GDAL.GDT_Int64 + @assert attributesizeref[] % sizeof(Int64) == 0 + nvalues = attributesizeref[] ÷ sizeof(Int64) + if dimensioncount == 0 + @assert nvalues == 1 + value = unsafe_load(Ptr{Int64}(valueptr)) + value::Int64 + else + value = + [unsafe_load(Ptr{Int64}(valueptr), n) for n in 1:nvalues] + value::Vector{Int64} + end + elseif numericdatatype === GDAL.GDT_Float64 + @assert attributesizeref[] % sizeof(Float64) == 0 + nvalues = attributesizeref[] ÷ sizeof(Float64) + if dimensioncount == 0 + @assert nvalues == 1 + value = unsafe_load(Ptr{Float64}(valueptr)) + value::Float64 + else + value = + [unsafe_load(Ptr{Float64}(valueptr), n) for n in 1:nvalues] + value::Vector{Float64} + end + else + @assert false + end + GDAL.gdalattributefreerawresult( + attributeh, + valueptr, + attributesizeref[], + ) + check_last_error() + else + @assert false + end + GDAL.gdalextendeddatatyperelease(datatypeh) + check_last_error() + GDAL.gdalattributerelease(attributeh) + check_last_error() + return value +end + +function setup() + version = GDAL.gdalversioninfo("") + println(version) + + # ndrivers = GDAL.gdalgetdrivercount() + # println("GDAL drivers:") + # for driver in 0:ndrivers-1 + # driverh = GDAL.gdalgetdriver(driver) + # drivershortname = GDAL.gdalgetdrivershortname(driverh) + # println("$driver: $drivershortname") + # end + + return nothing +end + +function create_file(drivername::AbstractString, path::AbstractString) + println("Creating file \"$(path)\" via $(drivername) driver...") + + # driverh = GDAL.gdalgetdriverbyname(drivername) + # check_last_error() + # @assert driverh != C_NULL + + driver = AG.getdriver(drivername) + check_last_error() + @assert !AG.isnull(driver) + + # driverlongname = GDAL.gdalgetdriverlongname(driverh) + + # drivercreationoptionlist = GDAL.gdalgetdrivercreationoptionlist(driverh) + # println(drivercreationoptionlist) + + rm(path; force = true, recursive = true) + + rootgroupoptions = String[] + if drivername == "HDF5" + createoptions = ["FORMAT=NC4"] + elseif drivername == "netCDF" + createoptions = ["FORMAT=NC4"] + elseif drivername == "Zarr" + createoptions = ["FORMAT=ZARR_V3"] + else + @assert false + createoptions = String[] + end + # rootgroupoptionptrs = Cstring[[pointer(opt) for opt in rootgroupoptions]; Cstring(C_NULL)] + # createoptionptrs = Cstring[[pointer(opt) for opt in createoptions]; Cstring(C_NULL)] + # dataseth = GDAL.gdalcreatemultidimensional(driverh, path, rootgroupoptionptrs, createoptionptrs) + # check_last_error() + # @assert dataseth != C_NULL + + dataset = + AG.createmultidimensional(driver, path, rootgroupoptions, createoptions) + @show typeof(dataset) + @show dataset + check_last_error() + @assert !AG.isnull(dataset) + + # grouph = GDAL.gdaldatasetgetrootgroup(dataset.ptr) + # check_last_error() + # @assert grouph != C_NULL + + group = AG.getrootgroup(dataset) + check_last_error() + @assert !AG.isnull(group) + + data = make_data() + #TODO dimensionnames = ["D", "P", "F", "T"] + dimensionnames = ["D", "P"] + @assert length(dimensionnames) == ndims(data) + + datatypeh = GDAL.gdalextendeddatatypecreate(GDAL.GDT_Float32) + check_last_error() + @assert datatypeh != C_NULL + + dimensionhs = GDAL.GDALDimensionH[ + let + dimh = GDAL.gdalgroupcreatedimension( + group.ptr, + dimensionnames[d], + "UNUSED", + "UNUSED", + size(data, d), + GDAL.CSLConstList(), + ) + check_last_error() + @assert dimh != C_NULL + dimh + end for d in ndims(data):-1:1 + ] + + # blocksize = join(reverse(size(data)), ",") + blocksize = "8192,1,2,64" + if drivername == "Zarr" + #TODO arrayoptions = ["COMPRESS=BLOSC", "BLOCKSIZE=$(blocksize)", "BLOSC_CLEVEL=9", "BLOSC_SHUFFLE=BIT"] + arrayoptions = [] + elseif drivername == "netCDF" + #TODO arrayoptions = ["COMPRESS=DEFLATE", "BLOCKSIZE=$(blocksize)", "ZLEVEL=9"] + arrayoptions = [] + else + @assert false + arrayoptions = [] + end + arrayoptionptrs = + Cstring[[pointer(opt) for opt in arrayoptions]; Cstring(C_NULL)] + + mdarrayh = GDAL.gdalgroupcreatemdarray( + group.ptr, + "data", + length(dimensionhs), + dimensionhs, + datatypeh, + arrayoptionptrs, + ) + check_last_error() + @assert mdarrayh != C_NULL + + # write_attribute(mdarrayh, "string_attribute", "hello, world!") + # write_attribute(mdarrayh, "int_attribute", 42) + # write_attribute(mdarrayh, "float_attribute", pi) + # write_attribute(mdarrayh, "int_array_attribute", [1, 2, 3]) + # write_attribute(mdarrayh, "float_array_attribute", [1.1, 1.2, 1.3]) + # write_attribute(mdarrayh, "large_array_attribute", collect(1:1000)) + + noerr = GDAL.gdalmdarraywrite( + mdarrayh, + #TODO GDAL.GUIntBig[0, 0, 0, 0], + GDAL.GUIntBig[0, 0], + GDAL.GUIntBig[size(data, d) for d in ndims(data):-1:1], + C_NULL, + C_NULL, + datatypeh, + data, + data, + sizeof(data), + ) + check_last_error() + @assert noerr == true + + # GDAL.gdalmdarrayrelease(mdarrayh) + # check_last_error() + + # GDAL.gdaldimensionrelease.(dimensionhs) + # check_last_error() + + # GDAL.gdalextendeddatatyperelease(datatypeh) + # check_last_error() + + # GDAL.gdalgrouprelease(group.ptr) + # check_last_error() + # group.ptr = C_NULL + + # AG.destroy(group) + + # err = GDAL.gdalclose(dataset.ptr) + # check_last_error() + # @assert err === GDAL.CE_None + + err = AG.close(dataset) + check_last_error() + @assert err === GDAL.CE_None + + AG.destroy(group) + + return nothing +end + +function read_file(path::AbstractString) + println("Reading file \"$(path)\"...") + + dataseth = GDAL.gdalopenex( + path, + GDAL.GDAL_OF_MULTIDIM_RASTER | + GDAL.GDAL_OF_READONLY | + GDAL.GDAL_OF_SHARED | + GDAL.GDAL_OF_VERBOSE_ERROR, + C_NULL, + C_NULL, + C_NULL, + ) + check_last_error() + @assert dataseth != C_NULL + + # info = GDAL.gdalinfo(dataseth, C_NULL) + # println("Info:\n", info) + + # multidiminfo = GDAL.gdalmultidiminfo(dataseth, C_NULL) + # println("Info:\n", multidiminfo) + + grouph = GDAL.gdaldatasetgetrootgroup(dataseth) + check_last_error() + @assert grouph != C_NULL + + arraynames = GDAL.gdalgroupgetmdarraynames(grouph, C_NULL) + check_last_error() + println("Array names:") + for arrayname in arraynames + println(" \"$(arrayname)\"") + end + + mdarrayh = GDAL.gdalgroupopenmdarray(grouph, "data", GDAL.CSLConstList()) + check_last_error() + @assert mdarrayh != C_NULL + + attributescountref = Ref(~Csize_t(0)) + attributehptr = + GDAL.gdalmdarraygetattributes(mdarrayh, attributescountref, C_NULL) + check_last_error() + attributehs = GDAL.GDALAttributeH[ + unsafe_load(attributehptr, d) for d in 1:attributescountref[] + ] + + attributenames = [ + let + name = GDAL.gdalattributegetname(attrh) + check_last_error() + name + end for attrh in attributehs + ] + println("Attribute names:") + for name in attributenames + println(" $(name)") + end + + GDAL.gdalreleaseattributes(attributehptr, attributescountref[]) + check_last_error() + + # @assert read_attribute(mdarrayh, "string_attribute") == "hello, world!" + # # Zarr stores Int64 attributes as Float64 because JSON... + # @assert read_attribute(mdarrayh, "int_attribute") == 42 + # @assert read_attribute(mdarrayh, "float_attribute") === Float64(pi) + # @assert read_attribute(mdarrayh, "int_array_attribute") == [1, 2, 3] + # @assert read_attribute(mdarrayh, "float_array_attribute") == [1.1, 1.2, 1.3] + # @assert read_attribute(mdarrayh, "large_array_attribute") == 1:1000 + + datatypeh = GDAL.gdalmdarraygetdatatype(mdarrayh) + check_last_error() + @assert datatypeh != C_NULL + + class = GDAL.gdalextendeddatatypegetclass(datatypeh) + check_last_error() + @assert class === GDAL.GEDTC_NUMERIC + + numericdatatype = GDAL.gdalextendeddatatypegetnumericdatatype(datatypeh) + check_last_error() + @assert numericdatatype === GDAL.GDT_Float32 + + totalelementscount = GDAL.gdalmdarraygettotalelementscount(mdarrayh) + check_last_error() + println("Total elements: $(totalelementscount)") + + dimensioncount = GDAL.gdalmdarraygetdimensioncount(mdarrayh) + check_last_error() + println("Rank: $(dimensioncount)") + + dimensionscountref = Ref(~Csize_t(0)) + dimensionhptr = GDAL.gdalmdarraygetdimensions(mdarrayh, dimensionscountref) + check_last_error() + dimensionhs = GDAL.GDALDimensionH[ + unsafe_load(dimensionhptr, d) for d in 1:dimensionscountref[] + ] + @assert length(dimensionhs) == dimensioncount + + dimensionnames = [ + let + name = GDAL.gdaldimensiongetname(dimh) + check_last_error() + name + end for dimh in reverse(dimensionhs) + ] + println("Dimension names: $(dimensionnames)") + + # for dimh in dimensionhs + # @show GDAL.gdaldimensiongetfullname(dimh) + # end + # for dimh in dimensionhs + # @show GDAL.gdaldimensiongettype(dimh) + # end + # for dimh in dimensionhs + # @show GDAL.gdaldimensiongetdirection(dimh) + # end + + sizes = Int[ + let + size = GDAL.gdaldimensiongetsize(dimh) + check_last_error() + size + end for dimh in reverse(dimensionhs) + ] + println("Size: $(sizes)") + @assert prod(sizes) == totalelementscount + + data = Array{Float32}(undef, sizes...) + + noerr = GDAL.gdalmdarrayread( + mdarrayh, + GDAL.GUIntBig[0, 0, 0, 0], + GDAL.GUIntBig[size(data, d) for d in ndims(data):-1:1], + C_NULL, + C_NULL, + datatypeh, + data, + data, + sizeof(data), + ) + check_last_error() + @assert noerr == true + + good_data = make_data() + @assert data == good_data + + GDAL.gdalmdarrayrelease(mdarrayh) + check_last_error() + GDAL.gdalreleasedimensions(dimensionhptr, dimensionscountref[]) + check_last_error() + GDAL.gdalextendeddatatyperelease(datatypeh) + check_last_error() + GDAL.gdalgrouprelease(grouph) + check_last_error() + + err = GDAL.gdalclose(dataseth) + check_last_error() + @assert err === GDAL.CE_None + + return nothing +end + +function main() + println("Experiment with Zarr files via the GDAL library") + drivername = "netCDF" + path = "/tmp/dataset.nc" + # drivername = "HDF5" + # path = "/tmp/dataset.h5" + # drivername = "Zarr" + # path = "/tmp/dataset.zarr" + setup() + create_file(drivername, path) + read_file(path) + println("Done.") + return nothing +end + +main() diff --git a/test/runtests.jl b/test/runtests.jl index 5fa1faf7..f5bffa0f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,51 +1,54 @@ using Test using Dates using GDAL -import ArchGDAL +#TODO import ArchGDAL +import ArchGDAL as AG import Aqua -# ensure all testing files are present -include("remotefiles.jl") +include("test_mdarray.jl") -@testset "ArchGDAL" begin - cd(dirname(@__FILE__)) do - isdir("tmp") || mkpath("tmp") - #TODO include("test_doctest.jl") - #TODO include("test_convert.jl") - #TODO include("test_tables.jl") - #TODO include("test_gdal_tutorials.jl") - #TODO include("test_geometry.jl") - #TODO include("test_types.jl") - #TODO include("test_display.jl") - #TODO include("test_drivers.jl") - #TODO include("test_feature.jl") - #TODO include("test_featurelayer.jl") - #TODO include("test_fielddefn.jl") - #TODO include("test_iterators.jl") - #TODO include("test_styletable.jl") - #TODO include("test_dataset.jl") - #TODO include("test_rasterband.jl") - #TODO include("test_rasterio.jl") - #TODO include("test_array.jl") - #TODO include("test_spatialref.jl") - #TODO include("test_gdalutilities.jl") - #TODO include("test_gdalutilities_errors.jl") - #TODO include("test_rasterattrtable.jl") - include("test_mdarray.jl") - #TODO include("test_ospy_examples.jl") - #TODO include("test_geos_operations.jl") - #TODO include("test_cookbook_geometry.jl") - #TODO include("test_cookbook_projection.jl") - #TODO include("test_geotransform.jl") - #TODO include("test_images.jl") - #TODO include("test_utils.jl") - #TODO include("test_prepared_geometry.jl") - #TODO Aqua.test_all( - #TODO ArchGDAL; - #TODO ambiguities = false, - #TODO stale_deps = false, - #TODO piracies = false, - #TODO ) - return nothing - end -end +#TODO # ensure all testing files are present +#TODO include("remotefiles.jl") +#TODO +#TODO @testset "ArchGDAL" begin +#TODO cd(dirname(@__FILE__)) do +#TODO isdir("tmp") || mkpath("tmp") +#TODO #TODO include("test_doctest.jl") +#TODO #TODO include("test_convert.jl") +#TODO #TODO include("test_tables.jl") +#TODO #TODO include("test_gdal_tutorials.jl") +#TODO #TODO include("test_geometry.jl") +#TODO #TODO include("test_types.jl") +#TODO #TODO include("test_display.jl") +#TODO #TODO include("test_drivers.jl") +#TODO #TODO include("test_feature.jl") +#TODO #TODO include("test_featurelayer.jl") +#TODO #TODO include("test_fielddefn.jl") +#TODO #TODO include("test_iterators.jl") +#TODO #TODO include("test_styletable.jl") +#TODO #TODO include("test_dataset.jl") +#TODO #TODO include("test_rasterband.jl") +#TODO #TODO include("test_rasterio.jl") +#TODO #TODO include("test_array.jl") +#TODO #TODO include("test_spatialref.jl") +#TODO #TODO include("test_gdalutilities.jl") +#TODO #TODO include("test_gdalutilities_errors.jl") +#TODO #TODO include("test_rasterattrtable.jl") +#TODO include("test_mdarray.jl") +#TODO #TODO include("test_ospy_examples.jl") +#TODO #TODO include("test_geos_operations.jl") +#TODO #TODO include("test_cookbook_geometry.jl") +#TODO #TODO include("test_cookbook_projection.jl") +#TODO #TODO include("test_geotransform.jl") +#TODO #TODO include("test_images.jl") +#TODO #TODO include("test_utils.jl") +#TODO #TODO include("test_prepared_geometry.jl") +#TODO #TODO Aqua.test_all( +#TODO #TODO ArchGDAL; +#TODO #TODO ambiguities = false, +#TODO #TODO stale_deps = false, +#TODO #TODO piracies = false, +#TODO #TODO ) +#TODO return nothing +#TODO end +#TODO end diff --git a/test/test_mdarray.jl b/test/test_mdarray.jl index 1b250ecf..6f21d9fc 100644 --- a/test/test_mdarray.jl +++ b/test/test_mdarray.jl @@ -1,38 +1,13 @@ using Test import ArchGDAL as AG -# TODO: Should this become part of ArchGDAL?u -Base.isempty(x::AG.AbstractAttribute) = x.ptr == C_NULL -Base.isempty(x::AG.AbstractDataset) = x.ptr == C_NULL -Base.isempty(x::AG.AbstractDimension) = x.ptr == C_NULL -Base.isempty(x::AG.AbstractEDTComponent) = x.ptr == C_NULL -Base.isempty(x::AG.AbstractExtendedDataType) = x.ptr == C_NULL -Base.isempty(x::AG.AbstractFeature) = x.ptr == C_NULL -Base.isempty(x::AG.AbstractFeatureDefn) = x.ptr == C_NULL -Base.isempty(x::AG.AbstractFeatureLayer) = x.ptr == C_NULL -Base.isempty(x::AG.AbstractFieldDefn) = x.ptr == C_NULL -Base.isempty(x::AG.AbstractGeomFieldDefn) = x.ptr == C_NULL -Base.isempty(x::AG.AbstractGeometry) = x.ptr == C_NULL -Base.isempty(x::AG.AbstractGroup) = x.ptr == C_NULL -Base.isempty(x::AG.AbstractMDArray) = x.ptr == C_NULL -Base.isempty(x::AG.AbstractRasterBand) = x.ptr == C_NULL -Base.isempty(x::AG.AbstractSpatialRef) = x.ptr == C_NULL -Base.isempty(x::AG.ColorTable) = x.ptr == C_NULL -Base.isempty(x::AG.CoordTransform) = x.ptr == C_NULL -Base.isempty(x::AG.Driver) = x.ptr == C_NULL -Base.isempty(x::AG.Field) = x.ptr == C_NULL -Base.isempty(x::AG.RasterAttrTable) = x.ptr == C_NULL -Base.isempty(x::AG.StyleManager) = x.ptr == C_NULL -Base.isempty(x::AG.StyleTable) = x.ptr == C_NULL -Base.isempty(x::AG.StyleTool) = x.ptr == C_NULL - # There should be more drivers... Anyone willing to update GDAL? # Possible drivers: at least HDF4, HDF5, TileDB const mdarray_drivers = [ ( drivername = "MEM", - drivercreateoptions = String[], - mdarraycreateoptions = String[], + drivercreateoptions = nothing, + mdarraycreateoptions = nothing, ), ( drivername = "netCDF", @@ -50,6 +25,345 @@ const mdarray_drivers = [ ), ] +@testset "test_group.jl" begin + @testset "$drivername" for ( + drivername, + drivercreateoptions, + mdarraycreateoptions, + ) in mdarray_drivers + driver = AG.getdriver(drivername) + + @testset "interactive" begin + filename = tempname(; cleanup = false) + memory_dataset = nothing + + @testset "writing" begin + dataset = AG.createmultidimensional( + driver, + filename, + nothing, + drivercreateoptions, + ) + @test !AG.isnull(dataset) + + files = AG.filelist(dataset) + if drivername in ["MEM"] + @test length(files) == 0 + elseif drivername in ["netCDF", "Zarr"] + @test length(files) == 1 + else + @assert false + end + + root = AG.getrootgroup(dataset) + @test !AG.isnull(root) + rootname = AG.getname(root) + @test rootname == "/" + rootfullname = AG.getfullname(root) + @test rootfullname == "/" + + group = AG.creategroup(root, "group") + @test !AG.isnull(group) + @test AG.getname(group) == "group" + @test AG.getfullname(group) == "/group" + + @test AG.getgroupnames(root) == ["group"] + @test AG.getgroupnames(group) == [] + + dimx = AG.createdimension(group, "x", "", "", 3) + @test !AG.isnull(dimx) + dimy = AG.createdimension(group, "y", "", "", 4) + @test !AG.isnull(dimy) + + @test AG.getdimensions(root) == [] + dimensions = AG.getdimensions(group) + @test length(dimensions) == 2 + + datatype = AG.extendeddatatypecreate(Float32) + @test !AG.isnull(datatype) + + mdarray = AG.createmdarray( + group, + "mdarray", + [dimx, dimy], + datatype, + mdarraycreateoptions, + ) + @test !AG.isnull(mdarray) + + @test AG.getmdarraynames(root) == [] + @test AG.getmdarraynames(group) == ["mdarray"] + + @test AG.getvectorlayernames(root) == [] + @test AG.getvectorlayernames(group) == [] + + @test AG.getstructuralinfo(group) == [] + + mdarray1 = AG.openmdarrayfromfullname(root, "/group/mdarray") + @test !AG.isnull(mdarray1) + @test AG.getfullname(mdarray1) == "/group/mdarray" + @test AG.isnull( + AG.openmdarrayfromfullname(root, "/group/doesnotexist"), + ) + + mdarray2 = AG.resolvemdarray(root, "mdarray", "") + @test !AG.isnull(mdarray2) + @test AG.getfullname(mdarray2) == "/group/mdarray" + @test AG.isnull(AG.resolvemdarray(root, "doesnotexist", "")) + + group1 = AG.opengroupfromfullname(root, "/group") + @test !AG.isnull(group1) + @test AG.getfullname(group1) == "/group" + @test AG.isnull(AG.opengroupfromfullname(root, "/doesnotexist")) + + # dimx1 = AG.opendimensionfromfullname(root, "group/x") + # @test !AG.isnull(dimx1) + # dimz = AG.opendimensionfromfullname(root, "group/z") + # @test AG.isnull(dimz) + + # TODO: + # - createvectorlayer + # - deletegroup + # - deletemdarary + # - openvecgtorlayer + # - rename + # - subsetdimensionfromselection + + if drivername != "MEM" + err = AG.close(dataset) + @test err == GDAL.CE_None + else + memory_dataset = dataset + end + + # Trigger all finalizers + for i in 1:10 + GC.gc() + end + end + + @testset "reading" begin + if drivername != "MEM" + dataset = AG.open( + filename, + AG.OF_MULTIDIM_RASTER | + AG.OF_READONLY | + AG.OF_SHARED | + AG.OF_VERBOSE_ERROR, + nothing, + nothing, + nothing, + ) + else + dataset = memory_dataset + end + @test !AG.isnull(dataset) + + root = AG.getrootgroup(dataset) + @test !AG.isnull(root) + + group = AG.opengroup(root, "group") + @test !AG.isnull(group) + + mdarray = AG.openmdarray(group, "mdarray") + @test !AG.isnull(mdarray) + + err = AG.close(dataset) + @test err == GDAL.CE_None + + # Trigger all finalizers + for i in 1:10 + GC.gc() + end + end + end + + @testset "context handlers" begin + filename = tempname(; cleanup = false) + + @testset "writing" begin + AG.createmultidimensional( + driver, + filename, + nothing, + drivercreateoptions, + ) do dataset + @test !AG.isnull(dataset) + + files = AG.filelist(dataset) + files = AG.filelist(dataset) + if drivername in ["MEM"] + @test length(files) == 0 + elseif drivername in ["netCDF", "Zarr"] + @test length(files) == 1 + else + @assert false + end + + AG.getrootgroup(dataset) do root + @test !AG.isnull(root) + rootname = AG.getname(root) + @test rootname == "/" + rootfullname = AG.getfullname(root) + @test rootfullname == "/" + + AG.creategroup(root, "group") do group + @test !AG.isnull(group) + @test AG.getname(group) == "group" + @test AG.getfullname(group) == "/group" + + @test AG.getgroupnames(root) == ["group"] + @test AG.getgroupnames(group) == [] + + AG.createdimension(group, "x", "", "", 3) do dimx + @test !AG.isnull(dimx) + AG.createdimension( + group, + "y", + "", + "", + 4, + ) do dimy + @test !AG.isnull(dimy) + + AG.getdimensions(root) do dims + @test dims == [] + end + AG.getdimensions(group) do dimensions + @test length(dimensions) == 2 + end + + datatype = + AG.extendeddatatypecreate(Float32) + @test !AG.isnull(datatype) + + AG.createmdarray( + group, + "mdarray", + [dimx, dimy], + datatype, + mdarraycreateoptions, + ) do mdarray + @test !AG.isnull(mdarray) + + @test AG.getmdarraynames(root) == [] + @test AG.getmdarraynames(group) == + ["mdarray"] + + @test AG.getvectorlayernames(root) == [] + @test AG.getvectorlayernames(group) == + [] + + @test AG.getstructuralinfo(group) == [] + + AG.openmdarrayfromfullname( + root, + "/group/mdarray", + ) do mdarray1 + @test !AG.isnull(mdarray1) + @test AG.getfullname(mdarray1) == + "/group/mdarray" + end + AG.openmdarrayfromfullname( + root, + "/group/doesnotexist", + ) do doesnotexist + @test AG.isnull(doesnotexist) + end + + AG.resolvemdarray( + root, + "mdarray", + "", + ) do mdarray2 + @test !AG.isnull(mdarray2) + @test AG.getfullname(mdarray2) == + "/group/mdarray" + end + AG.resolvemdarray( + root, + "doesnotexist", + "", + ) do doesnotexist + @test AG.isnull(doesnotexist) + end + + AG.opengroupfromfullname( + root, + "/group", + ) do group1 + @test !AG.isnull(group1) + @test AG.getfullname(group1) == + "/group" + end + AG.opengroupfromfullname( + root, + "/doesnotexist", + ) do doesnotexist + @test AG.isnull(doesnotexist) + end + + # dimx1 = AG.opendimensionfromfullname(root, "group/x") + # @test !AG.isnull(dimx1) + # dimz = AG.opendimensionfromfullname(root, "group/z") + # @test AG.isnull(dimz) + + # TODO: + # - createvectorlayer + # - deletegroup + # - deletemdarary + # - openvecgtorlayer + # - rename + # - subsetdimensionfromselection + end + end + end + end + end + end + + # Trigger all finalizers + for i in 1:10 + GC.gc() + end + end + + if drivername != "MEM" + @testset "reading" begin + AG.open( + filename, + AG.OF_MULTIDIM_RASTER | + AG.OF_READONLY | + AG.OF_SHARED | + AG.OF_VERBOSE_ERROR, + nothing, + nothing, + nothing, + ) do dataset + @test !AG.isnull(dataset) + + root = AG.getrootgroup(dataset) + @test !AG.isnull(root) + + AG.opengroup(root, "group") do group + @test !AG.isnull(group) + + AG.openmdarray(group, "mdarray") do mdarray + @test !AG.isnull(mdarray) + end + end + end + + # Trigger all finalizers + for i in 1:10 + GC.gc() + end + end + end + end + end +end + @testset "test_mdarray.jl" begin @testset "$drivername" for ( drivername, @@ -58,94 +372,207 @@ const mdarray_drivers = [ ) in mdarray_drivers driver = AG.getdriver(drivername) - filename = tempname() - dataset = AG.createmultidimensional( - driver, - filename, - String[], - drivercreateoptions, - ) - @test !isempty(dataset) - - files = AG.filelist(dataset) - if drivername in ["MEM"] - @test length(files) == 0 - elseif drivername in ["netCDF", "Zarr"] - @test length(files) == 1 - else - @assert false + @testset "interactive" begin + filename = tempname(; cleanup = false) + memory_dataset = nothing + + @testset "writing" begin + dataset = AG.createmultidimensional( + driver, + filename, + nothing, + drivercreateoptions, + ) + @test !AG.isnull(dataset) + + root = AG.getrootgroup(dataset) + @test !AG.isnull(root) + + dimx = AG.createdimension(root, "x", "", "", 3) + @test !AG.isnull(dimx) + dimy = AG.createdimension(root, "y", "", "", 4) + @test !AG.isnull(dimy) + + datatype = AG.extendeddatatypecreate(Float32) + @test !AG.isnull(datatype) + + mdarray = AG.createmdarray( + root, + "mdarray", + [dimx, dimy], + datatype, + mdarraycreateoptions, + ) + @test !AG.isnull(mdarray) + + # @test AG.iswritable(mdarray) + + data = Float32[100 * x + y for y in 1:4, x in 1:3] + + success = AG.write(mdarray, data) + @test success + + if drivername != "MEM" + err = AG.close(dataset) + @test err == GDAL.CE_None + else + memory_dataset = dataset + end + + # Trigger all finalizers + for i in 1:10 + GC.gc() + end + end + + @testset "reading" begin + if drivername != "MEM" + dataset = AG.open( + filename, + AG.OF_MULTIDIM_RASTER | + AG.OF_READONLY | + AG.OF_SHARED | + AG.OF_VERBOSE_ERROR, + nothing, + nothing, + nothing, + ) + else + dataset = memory_dataset + end + @test !AG.isnull(dataset) + + root = AG.getrootgroup(dataset) + @test !AG.isnull(root) + + mdarray = AG.openmdarray(root, "mdarray") + @test !AG.isnull(mdarray) + + # @test !AG.iswritable(mdarray) + + dims = AG.getdimensions(mdarray) + @test length(dims) == 2 + # TODO: Check name, length + + datatype = AG.getdatatype(mdarray) + @test !AG.isnull(datatype) + # TODO: Check class + + data = Array{Float32}(undef, 4, 3) + success = AG.read!(mdarray, data) + @test success + @test data == Float32[100 * x + y for y in 1:4, x in 1:3] + + err = AG.close(dataset) + @test err == GDAL.CE_None + + # Trigger all finalizers + for i in 1:10 + GC.gc() + end + end end - root = AG.getrootgroup(dataset) - @test !isempty(root) - rootname = AG.getname(root) - @test rootname == "/" - rootfullname = AG.getfullname(root) - @test rootfullname == "/" - - group = AG.creategroup(root, "group") - @test !isempty(group) - @test AG.getname(group) == "group" - @test AG.getfullname(group) == "/group" - - @test AG.getgroupnames(root) == ["group"] - @test AG.getgroupnames(group) == [] - - dimx = AG.createdimension(group, "x", "", "", 3) - @test !isempty(dimx) - dimy = AG.createdimension(group, "y", "", "", 4) - @test !isempty(dimy) - - @test AG.getdimensions(root) == [] - dimensions = AG.getdimensions(group) - @test length(dimensions) == 2 - - type = AG.extendeddatatypecreate(Float32) - @test !isempty(type) - - mdarray = AG.createmdarray( - group, - "mdarray", - [dimx, dimy], - type, - mdarraycreateoptions, - ) - @test !isempty(mdarray) - - @test AG.getmdarraynames(root) == [] - @test AG.getmdarraynames(group) == ["mdarray"] - - @test AG.getvectorlayernames(root) == [] - @test AG.getvectorlayernames(group) == [] - - @test AG.getstructuralinfo(group) == [] - - mdarray1 = AG.openmdarrayfromfullname(root, "/group/mdarray") - @test !isempty(mdarray1) - #TODO @test AG.getfullname(mdarray1) == "/group/mdarray" - @test isempty(AG.openmdarrayfromfullname(root, "/group/doesnotexist")) - - mdarray2 = AG.resolvemdarray(root, "mdarray", "") - @test !isempty(mdarray2) - #TODO @test AG.getfullname(mdarray2) == "/group/mdarray" - @test isempty(AG.resolvemdarray(root, "doesnotexist", "")) - - group1 = AG.opengroupfromfullname(root, "/group") - @test !isempty(group1) - @test AG.getfullname(group1) == "/group" - @test isempty(AG.opengroupfromfullname(root, "/doesnotexist")) - - # dimx1 = AG.opendimensionfromfullname(root, "group/x") - # @test !isempty(dimx1) - # dimz = AG.opendimensionfromfullname(root, "group/z") - # @test isempty(dimz) - - # TODO: - # - createvectorlayer - # - deletegroup - # - deletemdarary - # - openvecgtorlayer - # - rename - # - subsetdimensionfromselection + @testset "context handlers" begin + filename = tempname(; cleanup = false) + memory_dataset = nothing + + @testset "writing" begin + AG.createmultidimensional( + driver, + filename, + nothing, + drivercreateoptions, + ) do dataset + @test !AG.isnull(dataset) + + root = AG.getrootgroup(dataset) + @test !AG.isnull(root) + + AG.createdimension(root, "x", "", "", 3) do dimx + @test !AG.isnull(dimx) + AG.createdimension(root, "y", "", "", 4) do dimy + @test !AG.isnull(dimy) + + AG.extendeddatatypecreate(Float32) do datatype + @test !AG.isnull(datatype) + + AG.createmdarray( + root, + "mdarray", + [dimx, dimy], + datatype, + mdarraycreateoptions, + ) do mdarray + @test !AG.isnull(mdarray) + + # @test AG.iswritable(mdarray) + + data = Float32[ + 100 * x + y for y in 1:4, x in 1:3 + ] + + success = AG.write(mdarray, data) + @test success + end + end + end + end + end + + # Trigger all finalizers + for i in 1:10 + GC.gc() + end + end + + if drivername != "MEM" + @testset "reading" begin + AG.open( + filename, + AG.OF_MULTIDIM_RASTER | + AG.OF_READONLY | + AG.OF_SHARED | + AG.OF_VERBOSE_ERROR, + nothing, + nothing, + nothing, + ) do dataset + @test !AG.isnull(dataset) + + AG.getrootgroup(dataset) do root + @test !AG.isnull(root) + + AG.openmdarray(root, "mdarray") do mdarray + @test !AG.isnull(mdarray) + + # @test !AG.iswritable(mdarray) + + AG.getdimensions(mdarray) do dims + @test length(dims) == 2 + # TODO: Check name, length + + datatype = AG.getdatatype(mdarray) + @test !AG.isnull(datatype) + # TODO: Check class + + data = Array{Float32}(undef, 4, 3) + success = AG.read!(mdarray, data) + @test success + @test data == Float32[ + 100 * x + y for y in 1:4, x in 1:3 + ] + end + end + end + end + + # Trigger all finalizers + for i in 1:10 + GC.gc() + end + end + end + end end end From 4fbf31685028e83a532d971b24e571108ce77a36 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 24 Jul 2024 12:37:14 -0400 Subject: [PATCH 04/19] Remove file --- test/gdal.jl | 594 --------------------------------------------------- 1 file changed, 594 deletions(-) delete mode 100644 test/gdal.jl diff --git a/test/gdal.jl b/test/gdal.jl deleted file mode 100644 index c2b67966..00000000 --- a/test/gdal.jl +++ /dev/null @@ -1,594 +0,0 @@ -using Test -import ArchGDAL as AG - -using GDAL - -# using GDAL_jll -# `$(GDAL_jll.gdalinfo_path()) --format Zarr` |> run -# `$(GDAL_jll.gdalmdiminfo_exe()) data.zarr` |> run - -function check_last_error() - error = GDAL.cplgetlasterrormsg() - isempty(error) && return nothing - println(error) - throw(ErrorException(error)) - return nothing -end - -function make_data() - ndishes = 64 - npolarizations = 2 - #TODO nfrequencies = 384 - #TODO ntimes = 8192 - #TODO data = Float32[ - #TODO 1000 * t + 100 * f + 10 * p + 1 * d for d in 0:(ndishes - 1), p in 0:(npolarizations - 1), f in 0:(nfrequencies - 1), - #TODO t in 0:(ntimes - 1) - #TODO ] - data = Float32[ - 10 * p + 1 * d for d in 0:(ndishes-1), p in 0:(npolarizations-1) - ] - return data -end - -function write_attribute( - mdarrayh::GDAL.GDALMDArrayH, - name::AbstractString, - value::AbstractString, -) - datatypeh = GDAL.gdalextendeddatatypecreatestring(length(value)) - check_last_error() - @assert datatypeh != C_NULL - attributeh = GDAL.gdalmdarraycreateattribute( - mdarrayh, - name, - 0, - C_NULL, - datatypeh, - C_NULL, - ) - check_last_error() - @assert attributeh != C_NULL - noerr = GDAL.gdalattributewritestring(attributeh, value) - check_last_error() - @assert noerr == true - GDAL.gdalattributerelease(attributeh) - check_last_error() - GDAL.gdalextendeddatatyperelease(datatypeh) - check_last_error() - return nothing -end - -function write_attribute( - mdarrayh::GDAL.GDALMDArrayH, - name::AbstractString, - value::Integer, -) - datatypeh = GDAL.gdalextendeddatatypecreate(GDAL.GDT_Int64) - check_last_error() - @assert datatypeh != C_NULL - attributeh = GDAL.gdalmdarraycreateattribute( - mdarrayh, - name, - 0, - C_NULL, - datatypeh, - C_NULL, - ) - check_last_error() - @assert attributeh != C_NULL - noerr = - GDAL.gdalattributewriteraw(attributeh, Ref(Int64(value)), sizeof(Int64)) - check_last_error() - @assert noerr == true - GDAL.gdalattributerelease(attributeh) - check_last_error() - GDAL.gdalextendeddatatyperelease(datatypeh) - check_last_error() - return nothing -end - -function write_attribute( - mdarrayh::GDAL.GDALMDArrayH, - name::AbstractString, - value::Real, -) - datatypeh = GDAL.gdalextendeddatatypecreate(GDAL.GDT_Float64) - check_last_error() - @assert datatypeh != C_NULL - attributeh = GDAL.gdalmdarraycreateattribute( - mdarrayh, - name, - 0, - C_NULL, - datatypeh, - C_NULL, - ) - check_last_error() - @assert attributeh != C_NULL - noerr = GDAL.gdalattributewriteraw( - attributeh, - Ref(Float64(value)), - sizeof(Float64), - ) - check_last_error() - @assert noerr == true - GDAL.gdalattributerelease(attributeh) - check_last_error() - GDAL.gdalextendeddatatyperelease(datatypeh) - check_last_error() - return nothing -end - -function write_attribute( - mdarrayh::GDAL.GDALMDArrayH, - name::AbstractString, - values::AbstractArray{<:Integer}, -) - datatypeh = GDAL.gdalextendeddatatypecreate(GDAL.GDT_Int64) - check_last_error() - @assert datatypeh != C_NULL - rank = ndims(values) - sizes = collect(GDAL.GUInt64.(reverse(size(values)))) - attributeh = GDAL.gdalmdarraycreateattribute( - mdarrayh, - name, - rank, - sizes, - datatypeh, - C_NULL, - ) - check_last_error() - @assert attributeh != C_NULL - noerr = GDAL.gdalattributewriteraw( - attributeh, - Int64.(values), - sizeof(Int64) * length(values), - ) - check_last_error() - @assert noerr == true - GDAL.gdalattributerelease(attributeh) - check_last_error() - GDAL.gdalextendeddatatyperelease(datatypeh) - check_last_error() - return nothing -end - -function write_attribute( - mdarrayh::GDAL.GDALMDArrayH, - name::AbstractString, - values::AbstractArray{<:Real}, -) - datatypeh = GDAL.gdalextendeddatatypecreate(GDAL.GDT_Float64) - check_last_error() - @assert datatypeh != C_NULL - rank = ndims(values) - sizes = collect(GDAL.GUInt64.(reverse(size(values)))) - attributeh = GDAL.gdalmdarraycreateattribute( - mdarrayh, - name, - rank, - sizes, - datatypeh, - C_NULL, - ) - check_last_error() - @assert attributeh != C_NULL - noerr = GDAL.gdalattributewriteraw( - attributeh, - Float64.(values), - sizeof(Float64) * length(values), - ) - check_last_error() - @assert noerr == true - GDAL.gdalattributerelease(attributeh) - check_last_error() - GDAL.gdalextendeddatatyperelease(datatypeh) - check_last_error() - return nothing -end - -function read_attribute(mdarrayh::GDAL.GDALMDArrayH, name::AbstractString) - attributeh = GDAL.gdalmdarraygetattribute(mdarrayh, name) - check_last_error() - @assert attributeh != C_NULL - datatypeh = GDAL.gdalattributegetdatatype(attributeh) - check_last_error() - dimensioncount = GDAL.gdalattributegetdimensioncount(attributeh) - check_last_error() - @assert dimensioncount in (0, 1) - class = GDAL.gdalextendeddatatypegetclass(datatypeh) - check_last_error() - if class === GDAL.GEDTC_STRING - @assert dimensioncount == 0 - value = GDAL.gdalattributereadasstring(attributeh) - value::AbstractString - elseif class === GDAL.GEDTC_NUMERIC - attributesizeref = Ref(~Csize_t(0)) - valueptr = GDAL.gdalattributereadasraw(attributeh, attributesizeref) - numericdatatype = GDAL.gdalextendeddatatypegetnumericdatatype(datatypeh) - check_last_error() - if numericdatatype === GDAL.GDT_Int64 - @assert attributesizeref[] % sizeof(Int64) == 0 - nvalues = attributesizeref[] ÷ sizeof(Int64) - if dimensioncount == 0 - @assert nvalues == 1 - value = unsafe_load(Ptr{Int64}(valueptr)) - value::Int64 - else - value = - [unsafe_load(Ptr{Int64}(valueptr), n) for n in 1:nvalues] - value::Vector{Int64} - end - elseif numericdatatype === GDAL.GDT_Float64 - @assert attributesizeref[] % sizeof(Float64) == 0 - nvalues = attributesizeref[] ÷ sizeof(Float64) - if dimensioncount == 0 - @assert nvalues == 1 - value = unsafe_load(Ptr{Float64}(valueptr)) - value::Float64 - else - value = - [unsafe_load(Ptr{Float64}(valueptr), n) for n in 1:nvalues] - value::Vector{Float64} - end - else - @assert false - end - GDAL.gdalattributefreerawresult( - attributeh, - valueptr, - attributesizeref[], - ) - check_last_error() - else - @assert false - end - GDAL.gdalextendeddatatyperelease(datatypeh) - check_last_error() - GDAL.gdalattributerelease(attributeh) - check_last_error() - return value -end - -function setup() - version = GDAL.gdalversioninfo("") - println(version) - - # ndrivers = GDAL.gdalgetdrivercount() - # println("GDAL drivers:") - # for driver in 0:ndrivers-1 - # driverh = GDAL.gdalgetdriver(driver) - # drivershortname = GDAL.gdalgetdrivershortname(driverh) - # println("$driver: $drivershortname") - # end - - return nothing -end - -function create_file(drivername::AbstractString, path::AbstractString) - println("Creating file \"$(path)\" via $(drivername) driver...") - - # driverh = GDAL.gdalgetdriverbyname(drivername) - # check_last_error() - # @assert driverh != C_NULL - - driver = AG.getdriver(drivername) - check_last_error() - @assert !AG.isnull(driver) - - # driverlongname = GDAL.gdalgetdriverlongname(driverh) - - # drivercreationoptionlist = GDAL.gdalgetdrivercreationoptionlist(driverh) - # println(drivercreationoptionlist) - - rm(path; force = true, recursive = true) - - rootgroupoptions = String[] - if drivername == "HDF5" - createoptions = ["FORMAT=NC4"] - elseif drivername == "netCDF" - createoptions = ["FORMAT=NC4"] - elseif drivername == "Zarr" - createoptions = ["FORMAT=ZARR_V3"] - else - @assert false - createoptions = String[] - end - # rootgroupoptionptrs = Cstring[[pointer(opt) for opt in rootgroupoptions]; Cstring(C_NULL)] - # createoptionptrs = Cstring[[pointer(opt) for opt in createoptions]; Cstring(C_NULL)] - # dataseth = GDAL.gdalcreatemultidimensional(driverh, path, rootgroupoptionptrs, createoptionptrs) - # check_last_error() - # @assert dataseth != C_NULL - - dataset = - AG.createmultidimensional(driver, path, rootgroupoptions, createoptions) - @show typeof(dataset) - @show dataset - check_last_error() - @assert !AG.isnull(dataset) - - # grouph = GDAL.gdaldatasetgetrootgroup(dataset.ptr) - # check_last_error() - # @assert grouph != C_NULL - - group = AG.getrootgroup(dataset) - check_last_error() - @assert !AG.isnull(group) - - data = make_data() - #TODO dimensionnames = ["D", "P", "F", "T"] - dimensionnames = ["D", "P"] - @assert length(dimensionnames) == ndims(data) - - datatypeh = GDAL.gdalextendeddatatypecreate(GDAL.GDT_Float32) - check_last_error() - @assert datatypeh != C_NULL - - dimensionhs = GDAL.GDALDimensionH[ - let - dimh = GDAL.gdalgroupcreatedimension( - group.ptr, - dimensionnames[d], - "UNUSED", - "UNUSED", - size(data, d), - GDAL.CSLConstList(), - ) - check_last_error() - @assert dimh != C_NULL - dimh - end for d in ndims(data):-1:1 - ] - - # blocksize = join(reverse(size(data)), ",") - blocksize = "8192,1,2,64" - if drivername == "Zarr" - #TODO arrayoptions = ["COMPRESS=BLOSC", "BLOCKSIZE=$(blocksize)", "BLOSC_CLEVEL=9", "BLOSC_SHUFFLE=BIT"] - arrayoptions = [] - elseif drivername == "netCDF" - #TODO arrayoptions = ["COMPRESS=DEFLATE", "BLOCKSIZE=$(blocksize)", "ZLEVEL=9"] - arrayoptions = [] - else - @assert false - arrayoptions = [] - end - arrayoptionptrs = - Cstring[[pointer(opt) for opt in arrayoptions]; Cstring(C_NULL)] - - mdarrayh = GDAL.gdalgroupcreatemdarray( - group.ptr, - "data", - length(dimensionhs), - dimensionhs, - datatypeh, - arrayoptionptrs, - ) - check_last_error() - @assert mdarrayh != C_NULL - - # write_attribute(mdarrayh, "string_attribute", "hello, world!") - # write_attribute(mdarrayh, "int_attribute", 42) - # write_attribute(mdarrayh, "float_attribute", pi) - # write_attribute(mdarrayh, "int_array_attribute", [1, 2, 3]) - # write_attribute(mdarrayh, "float_array_attribute", [1.1, 1.2, 1.3]) - # write_attribute(mdarrayh, "large_array_attribute", collect(1:1000)) - - noerr = GDAL.gdalmdarraywrite( - mdarrayh, - #TODO GDAL.GUIntBig[0, 0, 0, 0], - GDAL.GUIntBig[0, 0], - GDAL.GUIntBig[size(data, d) for d in ndims(data):-1:1], - C_NULL, - C_NULL, - datatypeh, - data, - data, - sizeof(data), - ) - check_last_error() - @assert noerr == true - - # GDAL.gdalmdarrayrelease(mdarrayh) - # check_last_error() - - # GDAL.gdaldimensionrelease.(dimensionhs) - # check_last_error() - - # GDAL.gdalextendeddatatyperelease(datatypeh) - # check_last_error() - - # GDAL.gdalgrouprelease(group.ptr) - # check_last_error() - # group.ptr = C_NULL - - # AG.destroy(group) - - # err = GDAL.gdalclose(dataset.ptr) - # check_last_error() - # @assert err === GDAL.CE_None - - err = AG.close(dataset) - check_last_error() - @assert err === GDAL.CE_None - - AG.destroy(group) - - return nothing -end - -function read_file(path::AbstractString) - println("Reading file \"$(path)\"...") - - dataseth = GDAL.gdalopenex( - path, - GDAL.GDAL_OF_MULTIDIM_RASTER | - GDAL.GDAL_OF_READONLY | - GDAL.GDAL_OF_SHARED | - GDAL.GDAL_OF_VERBOSE_ERROR, - C_NULL, - C_NULL, - C_NULL, - ) - check_last_error() - @assert dataseth != C_NULL - - # info = GDAL.gdalinfo(dataseth, C_NULL) - # println("Info:\n", info) - - # multidiminfo = GDAL.gdalmultidiminfo(dataseth, C_NULL) - # println("Info:\n", multidiminfo) - - grouph = GDAL.gdaldatasetgetrootgroup(dataseth) - check_last_error() - @assert grouph != C_NULL - - arraynames = GDAL.gdalgroupgetmdarraynames(grouph, C_NULL) - check_last_error() - println("Array names:") - for arrayname in arraynames - println(" \"$(arrayname)\"") - end - - mdarrayh = GDAL.gdalgroupopenmdarray(grouph, "data", GDAL.CSLConstList()) - check_last_error() - @assert mdarrayh != C_NULL - - attributescountref = Ref(~Csize_t(0)) - attributehptr = - GDAL.gdalmdarraygetattributes(mdarrayh, attributescountref, C_NULL) - check_last_error() - attributehs = GDAL.GDALAttributeH[ - unsafe_load(attributehptr, d) for d in 1:attributescountref[] - ] - - attributenames = [ - let - name = GDAL.gdalattributegetname(attrh) - check_last_error() - name - end for attrh in attributehs - ] - println("Attribute names:") - for name in attributenames - println(" $(name)") - end - - GDAL.gdalreleaseattributes(attributehptr, attributescountref[]) - check_last_error() - - # @assert read_attribute(mdarrayh, "string_attribute") == "hello, world!" - # # Zarr stores Int64 attributes as Float64 because JSON... - # @assert read_attribute(mdarrayh, "int_attribute") == 42 - # @assert read_attribute(mdarrayh, "float_attribute") === Float64(pi) - # @assert read_attribute(mdarrayh, "int_array_attribute") == [1, 2, 3] - # @assert read_attribute(mdarrayh, "float_array_attribute") == [1.1, 1.2, 1.3] - # @assert read_attribute(mdarrayh, "large_array_attribute") == 1:1000 - - datatypeh = GDAL.gdalmdarraygetdatatype(mdarrayh) - check_last_error() - @assert datatypeh != C_NULL - - class = GDAL.gdalextendeddatatypegetclass(datatypeh) - check_last_error() - @assert class === GDAL.GEDTC_NUMERIC - - numericdatatype = GDAL.gdalextendeddatatypegetnumericdatatype(datatypeh) - check_last_error() - @assert numericdatatype === GDAL.GDT_Float32 - - totalelementscount = GDAL.gdalmdarraygettotalelementscount(mdarrayh) - check_last_error() - println("Total elements: $(totalelementscount)") - - dimensioncount = GDAL.gdalmdarraygetdimensioncount(mdarrayh) - check_last_error() - println("Rank: $(dimensioncount)") - - dimensionscountref = Ref(~Csize_t(0)) - dimensionhptr = GDAL.gdalmdarraygetdimensions(mdarrayh, dimensionscountref) - check_last_error() - dimensionhs = GDAL.GDALDimensionH[ - unsafe_load(dimensionhptr, d) for d in 1:dimensionscountref[] - ] - @assert length(dimensionhs) == dimensioncount - - dimensionnames = [ - let - name = GDAL.gdaldimensiongetname(dimh) - check_last_error() - name - end for dimh in reverse(dimensionhs) - ] - println("Dimension names: $(dimensionnames)") - - # for dimh in dimensionhs - # @show GDAL.gdaldimensiongetfullname(dimh) - # end - # for dimh in dimensionhs - # @show GDAL.gdaldimensiongettype(dimh) - # end - # for dimh in dimensionhs - # @show GDAL.gdaldimensiongetdirection(dimh) - # end - - sizes = Int[ - let - size = GDAL.gdaldimensiongetsize(dimh) - check_last_error() - size - end for dimh in reverse(dimensionhs) - ] - println("Size: $(sizes)") - @assert prod(sizes) == totalelementscount - - data = Array{Float32}(undef, sizes...) - - noerr = GDAL.gdalmdarrayread( - mdarrayh, - GDAL.GUIntBig[0, 0, 0, 0], - GDAL.GUIntBig[size(data, d) for d in ndims(data):-1:1], - C_NULL, - C_NULL, - datatypeh, - data, - data, - sizeof(data), - ) - check_last_error() - @assert noerr == true - - good_data = make_data() - @assert data == good_data - - GDAL.gdalmdarrayrelease(mdarrayh) - check_last_error() - GDAL.gdalreleasedimensions(dimensionhptr, dimensionscountref[]) - check_last_error() - GDAL.gdalextendeddatatyperelease(datatypeh) - check_last_error() - GDAL.gdalgrouprelease(grouph) - check_last_error() - - err = GDAL.gdalclose(dataseth) - check_last_error() - @assert err === GDAL.CE_None - - return nothing -end - -function main() - println("Experiment with Zarr files via the GDAL library") - drivername = "netCDF" - path = "/tmp/dataset.nc" - # drivername = "HDF5" - # path = "/tmp/dataset.h5" - # drivername = "Zarr" - # path = "/tmp/dataset.zarr" - setup() - create_file(drivername, path) - read_file(path) - println("Done.") - return nothing -end - -main() From b85b5d558d42e0ad4d93cbd5f5af935a9fb185a0 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 24 Jul 2024 12:38:12 -0400 Subject: [PATCH 05/19] Re-enable tests --- test/runtests.jl | 90 ++++++++++++++++++++++++------------------------ 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index f5bffa0f..079c573b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -7,48 +7,48 @@ import Aqua include("test_mdarray.jl") -#TODO # ensure all testing files are present -#TODO include("remotefiles.jl") -#TODO -#TODO @testset "ArchGDAL" begin -#TODO cd(dirname(@__FILE__)) do -#TODO isdir("tmp") || mkpath("tmp") -#TODO #TODO include("test_doctest.jl") -#TODO #TODO include("test_convert.jl") -#TODO #TODO include("test_tables.jl") -#TODO #TODO include("test_gdal_tutorials.jl") -#TODO #TODO include("test_geometry.jl") -#TODO #TODO include("test_types.jl") -#TODO #TODO include("test_display.jl") -#TODO #TODO include("test_drivers.jl") -#TODO #TODO include("test_feature.jl") -#TODO #TODO include("test_featurelayer.jl") -#TODO #TODO include("test_fielddefn.jl") -#TODO #TODO include("test_iterators.jl") -#TODO #TODO include("test_styletable.jl") -#TODO #TODO include("test_dataset.jl") -#TODO #TODO include("test_rasterband.jl") -#TODO #TODO include("test_rasterio.jl") -#TODO #TODO include("test_array.jl") -#TODO #TODO include("test_spatialref.jl") -#TODO #TODO include("test_gdalutilities.jl") -#TODO #TODO include("test_gdalutilities_errors.jl") -#TODO #TODO include("test_rasterattrtable.jl") -#TODO include("test_mdarray.jl") -#TODO #TODO include("test_ospy_examples.jl") -#TODO #TODO include("test_geos_operations.jl") -#TODO #TODO include("test_cookbook_geometry.jl") -#TODO #TODO include("test_cookbook_projection.jl") -#TODO #TODO include("test_geotransform.jl") -#TODO #TODO include("test_images.jl") -#TODO #TODO include("test_utils.jl") -#TODO #TODO include("test_prepared_geometry.jl") -#TODO #TODO Aqua.test_all( -#TODO #TODO ArchGDAL; -#TODO #TODO ambiguities = false, -#TODO #TODO stale_deps = false, -#TODO #TODO piracies = false, -#TODO #TODO ) -#TODO return nothing -#TODO end -#TODO end +# ensure all testing files are present +include("remotefiles.jl") + +@testset "ArchGDAL" begin + cd(dirname(@__FILE__)) do + isdir("tmp") || mkpath("tmp") + #TODO include("test_doctest.jl") + #TODO include("test_convert.jl") + #TODO include("test_tables.jl") + #TODO include("test_gdal_tutorials.jl") + #TODO include("test_geometry.jl") + #TODO include("test_types.jl") + #TODO include("test_display.jl") + #TODO include("test_drivers.jl") + #TODO include("test_feature.jl") + #TODO include("test_featurelayer.jl") + #TODO include("test_fielddefn.jl") + #TODO include("test_iterators.jl") + #TODO include("test_styletable.jl") + #TODO include("test_dataset.jl") + #TODO include("test_rasterband.jl") + #TODO include("test_rasterio.jl") + #TODO include("test_array.jl") + #TODO include("test_spatialref.jl") + #TODO include("test_gdalutilities.jl") + #TODO include("test_gdalutilities_errors.jl") + #TODO include("test_rasterattrtable.jl") + include("test_mdarray.jl") + #TODO include("test_ospy_examples.jl") + #TODO include("test_geos_operations.jl") + #TODO include("test_cookbook_geometry.jl") + #TODO include("test_cookbook_projection.jl") + #TODO include("test_geotransform.jl") + #TODO include("test_images.jl") + #TODO include("test_utils.jl") + #TODO include("test_prepared_geometry.jl") + #TODO Aqua.test_all( + #TODO ArchGDAL; + #TODO ambiguities = false, + #TODO stale_deps = false, + #TODO piracies = false, + #TODO ) + return nothing + end +end From 6b0721f4a81839d66ee277d47a1f54c7bdc31d0d Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 24 Jul 2024 13:24:06 -0400 Subject: [PATCH 06/19] Simplify tests --- src/ArchGDAL.jl | 1 + src/context.jl | 3 +- src/mdarray/dimension.jl | 57 +++++ src/mdarray/global.jl | 32 ++- test/test_mdarray.jl | 460 +++++++++++---------------------------- 5 files changed, 209 insertions(+), 344 deletions(-) create mode 100644 src/mdarray/dimension.jl diff --git a/src/ArchGDAL.jl b/src/ArchGDAL.jl index c35e5c58..9c30ccda 100644 --- a/src/ArchGDAL.jl +++ b/src/ArchGDAL.jl @@ -32,6 +32,7 @@ include("ogr/featuredefn.jl") include("ogr/fielddefn.jl") include("ogr/styletable.jl") include("mdarray/types.jl") +include("mdarray/dimension.jl") include("mdarray/global.jl") include("mdarray/group.jl") include("mdarray/mdarray.jl") diff --git a/src/context.jl b/src/context.jl index d720bc27..7d599661 100644 --- a/src/context.jl +++ b/src/context.jl @@ -241,6 +241,7 @@ for gdalfunc in ( :getgeom, :getgridded, :getindex, + :getindexingvariable, :getlayer, :getmask, :getmaskband, @@ -265,7 +266,7 @@ for gdalfunc in ( :newspatialref, :nextfeature, :open, - # :opendimensionfromfullname, + :opendimensionfromfullname, :opengroup, :opengroupfromfullname, :openmdarray, diff --git a/src/mdarray/dimension.jl b/src/mdarray/dimension.jl new file mode 100644 index 00000000..544a4401 --- /dev/null +++ b/src/mdarray/dimension.jl @@ -0,0 +1,57 @@ +# GDALDimension + +function getname(dimension::AbstractDimension)::AbstractString + @assert !isnull(dimension) + return GDAL.gdaldimensiongetname(dimension) +end + +function getfullname(dimension::AbstractDimension)::AbstractString + @assert !isnull(dimension) + return GDAL.gdaldimensiongetfullname(dimension) +end + +function gettype(dimension::AbstractDimension)::AbstractString + @assert !isnull(dimension) + return GDAL.gdaldimensiongettype(dimension) +end + +function getdirection(dimension::AbstractDimension)::AbstractString + @assert !isnull(dimension) + return GDAL.gdaldimensiongetdirection(dimension) +end + +function getsize(dimension::AbstractDimension)::Int + @assert !isnull(dimension) + return Int(GDAL.gdaldimensiongetsize(dimension)) +end + +function unsafe_getindexingvariable( + dimension::AbstractDimension, +)::AbstractMDArray + @assert !isnull(dimension) + return MDArray( + GDAL.gdaldimensiongetindexingvariable(dimension), + dimension.dataset.value, + ) +end + +function getindexingvariable(dimension::AbstractDimension)::AbstractMDArray + @assert !isnull(dimension) + return IMDArray( + GDAL.gdaldimensiongetindexingvariable(dimension), + dimension.dataset.value, + ) +end + +function setindexingvariable!( + dimension::AbstractDimension, + indexingvariable::AbstractMDArray, +)::Bool + @assert !isnull(dimension) + return GDAL.gdaldimensionsetindexingvariable(dimension, indexingvariable) +end + +function rename!(dimension::AbstractDimension, newname::AbstractString)::Bool + @assert !isnull(dimension) + return GDAL.gdaldimensionrename(dimension, newname) +end diff --git a/src/mdarray/global.jl b/src/mdarray/global.jl index 47dea33e..e79dc484 100644 --- a/src/mdarray/global.jl +++ b/src/mdarray/global.jl @@ -5,6 +5,8 @@ function unsafe_createmultidimensional( name::AbstractString, rootgroupoptions::OptionList = nothing, options::OptionList = nothing, + ; + hard_close::Bool = true, )::AbstractDataset @assert !isnull(driver) return Dataset( @@ -14,7 +16,7 @@ function unsafe_createmultidimensional( CSLConstListWrapper(rootgroupoptions), CSLConstListWrapper(options), ), - hard_close = true, + hard_close = hard_close, ) end @@ -23,6 +25,8 @@ function createmultidimensional( name::AbstractString, rootgroupoptions::OptionList = nothing, options::OptionList = nothing, + ; + hard_close::Bool = true, )::AbstractDataset @assert !isnull(driver) return IDataset( @@ -32,7 +36,7 @@ function createmultidimensional( CSLConstListWrapper(rootgroupoptions), CSLConstListWrapper(options), ), - hard_close = true, + hard_close = hard_close, ) end @@ -42,10 +46,14 @@ function unsafe_open( alloweddrivers::OptionList, openoptions::OptionList, siblingfiles::OptionList, + hard_close::Union{Nothing,Bool} = nothing, )::AbstractDataset - # We hard-close the dataset if it is a writable multidim dataset - want_hard_close = - (openflags & OF_MULTIDIM_RASTER != 0) && (openflags & OF_UPDATE != 0) + if hard_close === nothing + # We hard-close the dataset if it is a writable multidim dataset + hard_close = + (openflags & OF_MULTIDIM_RASTER != 0) && + (openflags & OF_UPDATE != 0) + end return Dataset( GDAL.gdalopenex( filename, @@ -54,7 +62,7 @@ function unsafe_open( CSLConstListWrapper(openoptions), CSLConstListWrapper(siblingfiles), ), - hard_close = want_hard_close, + hard_close = hard_close, ) end @@ -64,10 +72,14 @@ function open( alloweddrivers::OptionList, openoptions::OptionList, siblingfiles::OptionList, + hard_close::Union{Nothing,Bool} = nothing, )::AbstractDataset - # We hard-close the dataset if it is a writable multidim dataset - want_hard_close = - (openflags & OF_MULTIDIM_RASTER != 0) && (openflags & OF_UPDATE != 0) + if hard_close === nothing + # We hard-close the dataset if it is a writable multidim dataset + hard_close = + (openflags & OF_MULTIDIM_RASTER != 0) && + (openflags & OF_UPDATE != 0) + end return IDataset( GDAL.gdalopenex( filename, @@ -76,7 +88,7 @@ function open( CSLConstListWrapper(openoptions), CSLConstListWrapper(siblingfiles), ), - hard_close = want_hard_close, + hard_close = hard_close, ) end diff --git a/test/test_mdarray.jl b/test/test_mdarray.jl index 6f21d9fc..d773b96b 100644 --- a/test/test_mdarray.jl +++ b/test/test_mdarray.jl @@ -25,7 +25,7 @@ const mdarray_drivers = [ ), ] -@testset "test_group.jl" begin +@testset "test_mdarray.jl" begin @testset "$drivername" for ( drivername, drivercreateoptions, @@ -75,10 +75,6 @@ const mdarray_drivers = [ dimy = AG.createdimension(group, "y", "", "", 4) @test !AG.isnull(dimy) - @test AG.getdimensions(root) == [] - dimensions = AG.getdimensions(group) - @test length(dimensions) == 2 - datatype = AG.extendeddatatypecreate(Float32) @test !AG.isnull(datatype) @@ -99,35 +95,12 @@ const mdarray_drivers = [ @test AG.getstructuralinfo(group) == [] - mdarray1 = AG.openmdarrayfromfullname(root, "/group/mdarray") - @test !AG.isnull(mdarray1) - @test AG.getfullname(mdarray1) == "/group/mdarray" - @test AG.isnull( - AG.openmdarrayfromfullname(root, "/group/doesnotexist"), - ) - - mdarray2 = AG.resolvemdarray(root, "mdarray", "") - @test !AG.isnull(mdarray2) - @test AG.getfullname(mdarray2) == "/group/mdarray" - @test AG.isnull(AG.resolvemdarray(root, "doesnotexist", "")) - - group1 = AG.opengroupfromfullname(root, "/group") - @test !AG.isnull(group1) - @test AG.getfullname(group1) == "/group" - @test AG.isnull(AG.opengroupfromfullname(root, "/doesnotexist")) + # @test AG.iswritable(mdarray) - # dimx1 = AG.opendimensionfromfullname(root, "group/x") - # @test !AG.isnull(dimx1) - # dimz = AG.opendimensionfromfullname(root, "group/z") - # @test AG.isnull(dimz) + data = Float32[100 * x + y for y in 1:4, x in 1:3] - # TODO: - # - createvectorlayer - # - deletegroup - # - deletemdarary - # - openvecgtorlayer - # - rename - # - subsetdimensionfromselection + success = AG.write(mdarray, data) + @test success if drivername != "MEM" err = AG.close(dataset) @@ -168,6 +141,40 @@ const mdarray_drivers = [ mdarray = AG.openmdarray(group, "mdarray") @test !AG.isnull(mdarray) + # @test !AG.iswritable(mdarray) + + dims = AG.getdimensions(mdarray) + @test length(dims) == 2 + # TODO: Check name, length + + mdarray1 = AG.openmdarrayfromfullname(root, "/group/mdarray") + @test !AG.isnull(mdarray1) + @test AG.getfullname(mdarray1) == "/group/mdarray" + @test AG.isnull( + AG.openmdarrayfromfullname(root, "/group/doesnotexist"), + ) + + mdarray2 = AG.resolvemdarray(group, "mdarray", "") + @test !AG.isnull(mdarray2) + @test AG.getfullname(mdarray2) == "/group/mdarray" + @test AG.isnull(AG.resolvemdarray(group, "doesnotexist", "")) + + group1 = AG.opengroupfromfullname(root, "/group") + @test !AG.isnull(group1) + @test AG.getfullname(group1) == "/group" + @test AG.isnull( + AG.opengroupfromfullname(group, "/doesnotexist"), + ) + + datatype = AG.getdatatype(mdarray) + @test !AG.isnull(datatype) + # TODO: Check class + + data = Array{Float32}(undef, 4, 3) + success = AG.read!(mdarray, data) + @test success + @test data == Float32[100 * x + y for y in 1:4, x in 1:3] + err = AG.close(dataset) @test err == GDAL.CE_None @@ -180,6 +187,7 @@ const mdarray_drivers = [ @testset "context handlers" begin filename = tempname(; cleanup = false) + memory_dataset = nothing @testset "writing" begin AG.createmultidimensional( @@ -190,51 +198,35 @@ const mdarray_drivers = [ ) do dataset @test !AG.isnull(dataset) - files = AG.filelist(dataset) - files = AG.filelist(dataset) - if drivername in ["MEM"] - @test length(files) == 0 - elseif drivername in ["netCDF", "Zarr"] - @test length(files) == 1 - else - @assert false - end + root = AG.getrootgroup(dataset) + @test !AG.isnull(root) - AG.getrootgroup(dataset) do root - @test !AG.isnull(root) - rootname = AG.getname(root) - @test rootname == "/" - rootfullname = AG.getfullname(root) - @test rootfullname == "/" - - AG.creategroup(root, "group") do group - @test !AG.isnull(group) - @test AG.getname(group) == "group" - @test AG.getfullname(group) == "/group" - - @test AG.getgroupnames(root) == ["group"] - @test AG.getgroupnames(group) == [] - - AG.createdimension(group, "x", "", "", 3) do dimx - @test !AG.isnull(dimx) - AG.createdimension( - group, - "y", - "", - "", - 4, - ) do dimy - @test !AG.isnull(dimy) - - AG.getdimensions(root) do dims - @test dims == [] - end - AG.getdimensions(group) do dimensions - @test length(dimensions) == 2 - end + rootname = AG.getname(root) + @test rootname == "/" + rootfullname = AG.getfullname(root) + @test rootfullname == "/" + + AG.creategroup(root, "group") do group + @test !AG.isnull(group) + @test AG.getname(group) == "group" + @test AG.getfullname(group) == "/group" - datatype = - AG.extendeddatatypecreate(Float32) + @test AG.getgroupnames(root) == ["group"] + @test AG.getgroupnames(group) == [] + + AG.createdimension(group, "x", "", "", 3) do dimx + @test !AG.isnull(dimx) + AG.createdimension(group, "y", "", "", 4) do dimy + @test !AG.isnull(dimy) + + AG.getdimensions(root) do dims + @test dims == [] + end + AG.getdimensions(group) do dimensions + @test length(dimensions) == 2 + end + + AG.extendeddatatypecreate(Float32) do datatype @test !AG.isnull(datatype) AG.createmdarray( @@ -256,6 +248,60 @@ const mdarray_drivers = [ @test AG.getstructuralinfo(group) == [] + # @test AG.iswritable(mdarray) + + data = Float32[ + 100 * x + y for y in 1:4, x in 1:3 + ] + + success = AG.write(mdarray, data) + @test success + end + end + end + end + end + end + + # Trigger all finalizers + for i in 1:10 + GC.gc() + end + end + + if drivername != "MEM" + @testset "reading" begin + AG.open( + filename, + AG.OF_MULTIDIM_RASTER | + AG.OF_READONLY | + AG.OF_SHARED | + AG.OF_VERBOSE_ERROR, + nothing, + nothing, + nothing, + ) do dataset + @test !AG.isnull(dataset) + + AG.getrootgroup(dataset) do root + @test !AG.isnull(root) + + AG.opengroup(root, "group") do group + @test !AG.isnull(group) + + AG.openmdarray(group, "mdarray") do mdarray + @test !AG.isnull(mdarray) + + # @test !AG.iswritable(mdarray) + + AG.getdimensions(mdarray) do dims + @test length(dims) == 2 + # TODO: Check name, length + + datatype = AG.getdatatype(mdarray) + @test !AG.isnull(datatype) + # TODO: Check class + AG.openmdarrayfromfullname( root, "/group/mdarray", @@ -303,269 +349,17 @@ const mdarray_drivers = [ @test AG.isnull(doesnotexist) end - # dimx1 = AG.opendimensionfromfullname(root, "group/x") - # @test !AG.isnull(dimx1) - # dimz = AG.opendimensionfromfullname(root, "group/z") - # @test AG.isnull(dimz) - - # TODO: - # - createvectorlayer - # - deletegroup - # - deletemdarary - # - openvecgtorlayer - # - rename - # - subsetdimensionfromselection + data = Array{Float32}(undef, 4, 3) + success = AG.read!(mdarray, data) + @test success + @test data == Float32[ + 100 * x + y for y in 1:4, x in 1:3 + ] end end end end end - end - - # Trigger all finalizers - for i in 1:10 - GC.gc() - end - end - - if drivername != "MEM" - @testset "reading" begin - AG.open( - filename, - AG.OF_MULTIDIM_RASTER | - AG.OF_READONLY | - AG.OF_SHARED | - AG.OF_VERBOSE_ERROR, - nothing, - nothing, - nothing, - ) do dataset - @test !AG.isnull(dataset) - - root = AG.getrootgroup(dataset) - @test !AG.isnull(root) - - AG.opengroup(root, "group") do group - @test !AG.isnull(group) - - AG.openmdarray(group, "mdarray") do mdarray - @test !AG.isnull(mdarray) - end - end - end - - # Trigger all finalizers - for i in 1:10 - GC.gc() - end - end - end - end - end -end - -@testset "test_mdarray.jl" begin - @testset "$drivername" for ( - drivername, - drivercreateoptions, - mdarraycreateoptions, - ) in mdarray_drivers - driver = AG.getdriver(drivername) - - @testset "interactive" begin - filename = tempname(; cleanup = false) - memory_dataset = nothing - - @testset "writing" begin - dataset = AG.createmultidimensional( - driver, - filename, - nothing, - drivercreateoptions, - ) - @test !AG.isnull(dataset) - - root = AG.getrootgroup(dataset) - @test !AG.isnull(root) - - dimx = AG.createdimension(root, "x", "", "", 3) - @test !AG.isnull(dimx) - dimy = AG.createdimension(root, "y", "", "", 4) - @test !AG.isnull(dimy) - - datatype = AG.extendeddatatypecreate(Float32) - @test !AG.isnull(datatype) - - mdarray = AG.createmdarray( - root, - "mdarray", - [dimx, dimy], - datatype, - mdarraycreateoptions, - ) - @test !AG.isnull(mdarray) - - # @test AG.iswritable(mdarray) - - data = Float32[100 * x + y for y in 1:4, x in 1:3] - - success = AG.write(mdarray, data) - @test success - - if drivername != "MEM" - err = AG.close(dataset) - @test err == GDAL.CE_None - else - memory_dataset = dataset - end - - # Trigger all finalizers - for i in 1:10 - GC.gc() - end - end - - @testset "reading" begin - if drivername != "MEM" - dataset = AG.open( - filename, - AG.OF_MULTIDIM_RASTER | - AG.OF_READONLY | - AG.OF_SHARED | - AG.OF_VERBOSE_ERROR, - nothing, - nothing, - nothing, - ) - else - dataset = memory_dataset - end - @test !AG.isnull(dataset) - - root = AG.getrootgroup(dataset) - @test !AG.isnull(root) - - mdarray = AG.openmdarray(root, "mdarray") - @test !AG.isnull(mdarray) - - # @test !AG.iswritable(mdarray) - - dims = AG.getdimensions(mdarray) - @test length(dims) == 2 - # TODO: Check name, length - - datatype = AG.getdatatype(mdarray) - @test !AG.isnull(datatype) - # TODO: Check class - - data = Array{Float32}(undef, 4, 3) - success = AG.read!(mdarray, data) - @test success - @test data == Float32[100 * x + y for y in 1:4, x in 1:3] - - err = AG.close(dataset) - @test err == GDAL.CE_None - - # Trigger all finalizers - for i in 1:10 - GC.gc() - end - end - end - - @testset "context handlers" begin - filename = tempname(; cleanup = false) - memory_dataset = nothing - - @testset "writing" begin - AG.createmultidimensional( - driver, - filename, - nothing, - drivercreateoptions, - ) do dataset - @test !AG.isnull(dataset) - - root = AG.getrootgroup(dataset) - @test !AG.isnull(root) - - AG.createdimension(root, "x", "", "", 3) do dimx - @test !AG.isnull(dimx) - AG.createdimension(root, "y", "", "", 4) do dimy - @test !AG.isnull(dimy) - - AG.extendeddatatypecreate(Float32) do datatype - @test !AG.isnull(datatype) - - AG.createmdarray( - root, - "mdarray", - [dimx, dimy], - datatype, - mdarraycreateoptions, - ) do mdarray - @test !AG.isnull(mdarray) - - # @test AG.iswritable(mdarray) - - data = Float32[ - 100 * x + y for y in 1:4, x in 1:3 - ] - - success = AG.write(mdarray, data) - @test success - end - end - end - end - end - - # Trigger all finalizers - for i in 1:10 - GC.gc() - end - end - - if drivername != "MEM" - @testset "reading" begin - AG.open( - filename, - AG.OF_MULTIDIM_RASTER | - AG.OF_READONLY | - AG.OF_SHARED | - AG.OF_VERBOSE_ERROR, - nothing, - nothing, - nothing, - ) do dataset - @test !AG.isnull(dataset) - - AG.getrootgroup(dataset) do root - @test !AG.isnull(root) - - AG.openmdarray(root, "mdarray") do mdarray - @test !AG.isnull(mdarray) - - # @test !AG.iswritable(mdarray) - - AG.getdimensions(mdarray) do dims - @test length(dims) == 2 - # TODO: Check name, length - - datatype = AG.getdatatype(mdarray) - @test !AG.isnull(datatype) - # TODO: Check class - - data = Array{Float32}(undef, 4, 3) - success = AG.read!(mdarray, data) - @test success - @test data == Float32[ - 100 * x + y for y in 1:4, x in 1:3 - ] - end - end - end - end # Trigger all finalizers for i in 1:10 From 4c107fb1217664e263d7f4730242c69d32d47a2a Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 24 Jul 2024 13:48:30 -0400 Subject: [PATCH 07/19] Add Dimension tests --- test/test_mdarray.jl | 63 +++++++++++++++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 16 deletions(-) diff --git a/test/test_mdarray.jl b/test/test_mdarray.jl index d773b96b..e1fee60a 100644 --- a/test/test_mdarray.jl +++ b/test/test_mdarray.jl @@ -70,9 +70,10 @@ const mdarray_drivers = [ @test AG.getgroupnames(root) == ["group"] @test AG.getgroupnames(group) == [] - dimx = AG.createdimension(group, "x", "", "", 3) + nx, ny = 3, 4 + dimx = AG.createdimension(group, "x", "", "", nx) @test !AG.isnull(dimx) - dimy = AG.createdimension(group, "y", "", "", 4) + dimy = AG.createdimension(group, "y", "", "", ny) @test !AG.isnull(dimy) datatype = AG.extendeddatatypecreate(Float32) @@ -97,7 +98,7 @@ const mdarray_drivers = [ # @test AG.iswritable(mdarray) - data = Float32[100 * x + y for y in 1:4, x in 1:3] + data = Float32[100 * x + y for y in 1:ny, x in 1:nx] success = AG.write(mdarray, data) @test success @@ -143,9 +144,21 @@ const mdarray_drivers = [ # @test !AG.iswritable(mdarray) - dims = AG.getdimensions(mdarray) - @test length(dims) == 2 - # TODO: Check name, length + dimensions = AG.getdimensions(mdarray) + @test length(dimensions) == 2 + dimx, dimy = dimensions + @test all(!AG.isnull(dim) for dim in dimensions) + @test AG.getname(dimx) == "x" + @test AG.getname(dimy) == "y" + nx, ny = AG.getsize(dimx), AG.getsize(dimy) + @test (nx, ny) == (3, 4) + @test AG.getfullname(dimx) == "/group/x" + @test AG.gettype(dimx) == "" + @test AG.getdirection(dimx) == "" + xvar = AG.getindexingvariable(dimx) + @test AG.isnull(xvar) + # TODO: setindexingvariable! + # TODO: rename! mdarray1 = AG.openmdarrayfromfullname(root, "/group/mdarray") @test !AG.isnull(mdarray1) @@ -170,10 +183,10 @@ const mdarray_drivers = [ @test !AG.isnull(datatype) # TODO: Check class - data = Array{Float32}(undef, 4, 3) + data = Array{Float32}(undef, ny, nx) success = AG.read!(mdarray, data) @test success - @test data == Float32[100 * x + y for y in 1:4, x in 1:3] + @test data == Float32[100 * x + y for y in 1:ny, x in 1:nx] err = AG.close(dataset) @test err == GDAL.CE_None @@ -214,9 +227,10 @@ const mdarray_drivers = [ @test AG.getgroupnames(root) == ["group"] @test AG.getgroupnames(group) == [] - AG.createdimension(group, "x", "", "", 3) do dimx + nx, ny = 3, 4 + AG.createdimension(group, "x", "", "", nx) do dimx @test !AG.isnull(dimx) - AG.createdimension(group, "y", "", "", 4) do dimy + AG.createdimension(group, "y", "", "", ny) do dimy @test !AG.isnull(dimy) AG.getdimensions(root) do dims @@ -251,7 +265,7 @@ const mdarray_drivers = [ # @test AG.iswritable(mdarray) data = Float32[ - 100 * x + y for y in 1:4, x in 1:3 + 100 * x + y for y in 1:ny, x in 1:nx ] success = AG.write(mdarray, data) @@ -294,9 +308,26 @@ const mdarray_drivers = [ # @test !AG.iswritable(mdarray) - AG.getdimensions(mdarray) do dims - @test length(dims) == 2 - # TODO: Check name, length + AG.getdimensions(mdarray) do dimensions + @test length(dimensions) == 2 + dimx, dimy = dimensions + @test all( + !AG.isnull(dim) for + dim in dimensions + ) + @test AG.getname(dimx) == "x" + @test AG.getname(dimy) == "y" + nx, ny = + AG.getsize(dimx), AG.getsize(dimy) + @test (nx, ny) == (3, 4) + @test AG.getfullname(dimx) == "/group/x" + @test AG.gettype(dimx) == "" + @test AG.getdirection(dimx) == "" + AG.getindexingvariable(dimx) do xvar + @test AG.isnull(xvar) + end + # TODO: setindexingvariable! + # TODO: rename! datatype = AG.getdatatype(mdarray) @test !AG.isnull(datatype) @@ -349,11 +380,11 @@ const mdarray_drivers = [ @test AG.isnull(doesnotexist) end - data = Array{Float32}(undef, 4, 3) + data = Array{Float32}(undef, ny, nx) success = AG.read!(mdarray, data) @test success @test data == Float32[ - 100 * x + y for y in 1:4, x in 1:3 + 100 * x + y for y in 1:ny, x in 1:nx ] end end From a8e15a076a63e8f64d07c818e7d3779e73c673fd Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 24 Jul 2024 13:52:19 -0400 Subject: [PATCH 08/19] CI: Run benchmarks with Julia lts --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 9e3c5078..beed13c5 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -9,7 +9,7 @@ jobs: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@latest with: - version: 1.6 + version: lts - uses: julia-actions/julia-buildpkg@latest - name: Install dependencies run: julia -e 'using Pkg; pkg"add PkgBenchmark BenchmarkCI@0.1"' From 2292bdd906e8eb58ae5f0503a2f4637e335d48bb Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 24 Jul 2024 13:53:38 -0400 Subject: [PATCH 09/19] CI: Run benchmarks on macos-13 --- .github/workflows/benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index beed13c5..3f677d4f 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -4,12 +4,12 @@ on: [pull_request] jobs: Benchmark: - runs-on: macos-latest + runs-on: macos-13 steps: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@latest with: - version: lts + version: 1.6 - uses: julia-actions/julia-buildpkg@latest - name: Install dependencies run: julia -e 'using Pkg; pkg"add PkgBenchmark BenchmarkCI@0.1"' From 5e3980753cb74278c54225f3033b61a00dc7b522 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 24 Jul 2024 14:48:53 -0400 Subject: [PATCH 10/19] Add Attribute --- src/ArchGDAL.jl | 1 + src/context.jl | 3 + src/mdarray/attribute.jl | 401 +++++++++++++++++++++++++++++++++++++++ src/mdarray/group.jl | 1 - src/mdarray/types.jl | 1 + test/runtests.jl | 2 - test/test_mdarray.jl | 2 + 7 files changed, 408 insertions(+), 3 deletions(-) create mode 100644 src/mdarray/attribute.jl diff --git a/src/ArchGDAL.jl b/src/ArchGDAL.jl index 9c30ccda..4011c1ad 100644 --- a/src/ArchGDAL.jl +++ b/src/ArchGDAL.jl @@ -32,6 +32,7 @@ include("ogr/featuredefn.jl") include("ogr/fielddefn.jl") include("ogr/styletable.jl") include("mdarray/types.jl") +include("mdarray/attribute.jl") include("mdarray/dimension.jl") include("mdarray/global.jl") include("mdarray/group.jl") diff --git a/src/context.jl b/src/context.jl index 7d599661..8a695e6b 100644 --- a/src/context.jl +++ b/src/context.jl @@ -192,6 +192,7 @@ for gdalfunc in ( :copy, :create, :createRAT, + :createattribute, :createcolortable, :createcoordtrans, :createdimension, @@ -233,6 +234,8 @@ for gdalfunc in ( :gdaltranslate, :gdalvectortranslate, :gdalwarp, + :getattribute, + :getattributes, :getband, :getcolortable, :getdatatype, diff --git a/src/mdarray/attribute.jl b/src/mdarray/attribute.jl new file mode 100644 index 00000000..c7195e7f --- /dev/null +++ b/src/mdarray/attribute.jl @@ -0,0 +1,401 @@ +# GDALAttribute + +function getdimensionssize(attribute::AbstractAttribute)::NTuple{<:Any,Int} + @assert !isnull(attribute) + count = Ref{Csize_t}() + sizeptr = GDAL.gdalattributegetdimensionssize(attribute, count) + size = ntuple(n -> unsafe_load(sizeptr, n), count[]) + GDAL.vsifree(sizeptr) + return Int.(size) +end + +function readasraw(attribute::AbstractAttribute)::AbstractVector{UInt8} + @assert !isnull(attribute) + count = Ref{Csize_t}() + rawptr = GDAL.gdalattributereadasraw(attribute, count) + raw = UInt8[unsafe_load(rawptr, n) for n in 1:count[]] + GDAL.gdalattributefreerawresult(rawptr, count[]) + return raw +end + +function readasstring(attribute::AbstractAttribute)::String + @assert !isnull(attribute) + stringptr = GDAL.gdalattributereadasstring(attribute) + string = unsafe_string(stringptr) + # do not free + return string +end + +function readasint(attribute::AbstractAttribute)::Int32 + @assert !isnull(attribute) + return GDAL.gdalattributereadasint(attribute) +end + +function readasint64(attribute::AbstractAttribute)::Int64 + @assert !isnull(attribute) + return GDAL.gdalattributereadasint64(attribute) +end + +function readasdouble(attribute::AbstractAttribute)::Float64 + @assert !isnull(attribute) + return GDAL.gdalattributereadasdouble(attribute) +end + +function readasstringarray(attribute::AbstractAttribute)::Vector{String} + @assert !isnull(attribute) + return GDAL.gdalattributereadasstringarray(attribute) +end + +function readasintarray(attribute::AbstractAttribute)::Vector{Int32} + @assert !isnull(attribute) + count = Ref{Csize_t}() + ptr = GDAL.gdalattributereadasintarray(attribute, count) + ptr == C_NULL && return Int32[] + values = Int32[unsafe_load(ptr, n) for n in 1:count[]] + GDAL.vsifree(ptr) + return values +end + +function readasint64array(attribute::AbstractAttribute)::Vector{Int64} + @assert !isnull(attribute) + count = Ref{Csize_t}() + ptr = GDAL.gdalattributereadasint64array(attribute, count) + ptr == C_NULL && return Int64[] + values = Int64[unsafe_load(ptr, n) for n in 1:count[]] + GDAL.vsifree(ptr) + return values +end + +function readasdoublearray(attribute::AbstractAttribute)::Vector{Float64} + @assert !isnull(attribute) + count = Ref{Csize_t}() + ptr = GDAL.gdalattributereadasdoublearray(attribute, count) + ptr == C_NULL && return Float64[] + values = Float64[unsafe_load(ptr, n) for n in 1:count[]] + GDAL.vsifree(ptr) + return values +end + +read(::Type{String}, attribute)::String = readasstring(attribute) +read(::Type{Int32}, attribute)::Int32 = readasint(attribute) +read(::Type{Int64}, attribute)::Int64 = readasint64(attribute) +read(::Type{Float64}, attribute)::Float64 = readasdouble(attribute) +read(::Type{Vector{String}}, attribute)::Vector{String} = + readasstringarray(attribute) +read(::Type{Vector{Int32}}, attribute)::Vector{Int32} = + readasintarray(attribute) +read(::Type{Vector{Int64}}, attribute)::Vector{Int64} = + readasint64array(attribute) +read(::Type{Vector{Float64}}, attribute)::Vector{Float64} = + readasdoublearray(attribute) + +function writerraw( + attribute::AbstractAttribute, + value::AbstractVector{UInt8}, +)::Bool + @assert !isnull(attribute) + return Bool(GDAL.gdalattributewriteraw(attribute, value, length(value))) +end + +function write(attribute::AbstractAttribute, value::AbstractString)::Bool + @assert !isnull(attribute) + return Bool(GDAL.gdalattributewritestring(attribute, value)) +end + +function write(attribute::AbstractAttribute, value::Int32)::Bool + @assert !isnull(attribute) + return Bool(GDAL.gdalattributewriteint(attribute, value)) +end + +function write(attribute::AbstractAttribute, value::Int64)::Bool + @assert !isnull(attribute) + return Bool(GDAL.gdalattributewriteint64(attribute, value)) +end + +function write(attribute::AbstractAttribute, value::Float64)::Bool + @assert !isnull(attribute) + return Bool(GDAL.gdalattributewritedouble(attribute, value)) +end + +function write( + attribute::AbstractAttribute, + values::AbstractVector{<:AbstractString}, +)::Bool + @assert !isnull(attribute) + return Bool( + GDAL.gdalattributewritestringarray( + attribute, + CSLConstListWrapper(values), + ), + ) +end + +function write( + attribute::AbstractAttribute, + values::AbstractVector{Int32}, +)::Bool + @assert !isnull(attribute) + return Bool( + GDAL.gdalattributewriteintarray(attribute, values, length(values)), + ) +end + +function write( + attribute::AbstractAttribute, + values::AbstractVector{Int64}, +)::Bool + @assert !isnull(attribute) + return Bool( + GDAL.gdalattributewriteint64array(attribute, values, length(values)), + ) +end + +function write( + attribute::AbstractAttribute, + values::AbstractVector{Float64}, +)::Bool + @assert !isnull(attribute) + return Bool( + GDAL.gdalattributewritedoublearray(attribute, values, length(values)), + ) +end + +################################################################################ + +# For GDALGroup + +function unsafe_getattribute( + group::AbstractGroup, + name::AbstractString, +)::AbstractAttribute + @assert !isnull(group) + return Attribute( + GDAL.gdalgroupgetattribute(group, name), + group.dataset.value, + ) +end + +function getattribute( + group::AbstractGroup, + name::AbstractString, +)::AbstractAttribute + @assert !isnull(group) + return IAttribute( + GDAL.gdalgroupgetattribute(group, name), + group.dataset.value, + ) +end + +function unsafe_getattributes( + group::AbstractGroup, + options::OptionList = nothing, +)::AbstractVector{<:AbstractAttribute} + @assert !isnull(group) + count = Ref{Csize_t}() + ptr = + GDAL.gdalgroupgetattributes(group, count, CSLConstListWrapper(options)) + dataset = group.dataset.value + attributes = AbstractAttribute[ + Attribute(unsafe_load(ptr, n), dataset) for n in 1:count[] + ] + GDAL.vsifree(ptr) + return attributes +end + +function getattributes( + group::AbstractGroup, + options::OptionList = nothing, +)::AbstractVector{<:AbstractAttribute} + @assert !isnull(group) + count = Ref{Csize_t}() + ptr = + GDAL.gdalgroupgetattributes(group, count, CSLConstListWrapper(options)) + dataset = group.dataset.value + attributes = AbstractAttribute[ + IAttribute(unsafe_load(ptr, n), dataset) for n in 1:count[] + ] + GDAL.vsifree(ptr) + return attributes +end + +function unsafe_createattribute( + group::AbstractGroup, + name::AbstractString, + dimensions::AbstractVector{<:AbstractDimension}, + datatype::AbstractExtendedDataType, + options::OptionList = nothing, +)::AbstractAttribute + @assert !isnull(group) + @assert all(!isnull(dim) for dim in dimensions) + @assert !isnull(datatype) + return Attribute( + GDAL.gdalgroupcreateattribute( + group, + name, + length(dimensions), + DimensionHList(dimensions), + datatype, + CSLConstListWrapper(options), + ), + group.dataset.value, + ) +end + +function createattribute( + group::AbstractGroup, + name::AbstractString, + dimensions::AbstractVector{<:AbstractDimension}, + datatype::AbstractExtendedDataType, + options::OptionList = nothing, +)::AbstractAttribute + @assert !isnull(group) + @assert all(!isnull(dim) for dim in dimensions) + @assert !isnull(datatype) + return IAttribute( + GDAL.gdalgroupcreateattribute( + group, + name, + length(dimensions), + DimensionHList(dimensions), + datatype, + CSLConstListWrapper(options), + ), + group.dataset.value, + ) +end + +function deleteattribute( + group::AbstractGroup, + name::AbstractString, + options::OptionList = nothing, +)::Bool + @assert !isnull(group) + return GDAL.gdalgroupdeleteattribute( + group, + name, + CSLConstListWrapper(options), + ) +end + +################################################################################ + +# For GDALMDArray + +function unsafe_getattribute( + mdarray::AbstractMDArray, + name::AbstractString, +)::AbstractAttribute + @assert !isnull(mdarray) + return Attribute( + GDAL.gdalmdarraygetattribute(mdarray, name), + mdarray.dataset.value, + ) +end + +function getattribute( + mdarray::AbstractMDArray, + name::AbstractString, +)::AbstractAttribute + @assert !isnull(mdarray) + return IAttribute( + GDAL.gdalmdarraygetattribute(mdarray, name), + mdarray.dataset.value, + ) +end + +function unsafe_getattributes( + mdarray::AbstractMDArray, + options::OptionList = nothing, +)::AbstractVector{<:AbstractAttribute} + @assert !isnull(mdarray) + count = Ref{Csize_t}() + ptr = GDAL.gdalmdarraygetattributes( + mdarray, + count, + CSLConstListWrapper(options), + ) + dataset = mdarray.dataset.value + attributes = AbstractAttribute[ + Attribute(unsafe_load(ptr, n), dataset) for n in 1:count[] + ] + GDAL.vsifree(ptr) + return attributes +end + +function getattributes( + mdarray::AbstractMDArray, + options::OptionList = nothing, +)::AbstractVector{<:AbstractAttribute} + @assert !isnull(mdarray) + count = Ref{Csize_t}() + ptr = GDAL.gdalmdarraygetattributes( + mdarray, + count, + CSLConstListWrapper(options), + ) + dataset = mdarray.dataset.value + attributes = AbstractAttribute[ + IAttribute(unsafe_load(ptr, n), dataset) for n in 1:count[] + ] + GDAL.vsifree(ptr) + return attributes +end + +function unsafe_createattribute( + mdarray::AbstractMDArray, + name::AbstractString, + dimensions::AbstractVector{<:AbstractDimension}, + datatype::AbstractExtendedDataType, + options::OptionList = nothing, +)::AbstractAttribute + @assert !isnull(mdarray) + @assert all(!isnull(dim) for dim in dimensions) + @assert !isnull(datatype) + return Attribute( + GDAL.gdalmdarraycreateattribute( + mdarray, + name, + length(dimensions), + DimensionHList(dimensions), + datatype, + CSLConstListWrapper(options), + ), + mdarray.dataset.value, + ) +end + +function createattribute( + mdarray::AbstractMDArray, + name::AbstractString, + dimensions::AbstractVector{<:AbstractDimension}, + datatype::AbstractExtendedDataType, + options::OptionList = nothing, +)::AbstractAttribute + @assert !isnull(mdarray) + @assert all(!isnull(dim) for dim in dimensions) + @assert !isnull(datatype) + return IAttribute( + GDAL.gdalmdarraycreateattribute( + mdarray, + name, + length(dimensions), + DimensionHList(dimensions), + datatype, + CSLConstListWrapper(options), + ), + mdarray.dataset.value, + ) +end + +function deleteattribute( + mdarray::AbstractMDArray, + name::AbstractString, + options::OptionList = nothing, +)::Bool + @assert !isnull(mdarray) + return GDAL.gdalmdarraydeleteattribute( + mdarray, + name, + CSLConstListWrapper(options), + ) +end diff --git a/src/mdarray/group.jl b/src/mdarray/group.jl index 83bad039..98d4deac 100644 --- a/src/mdarray/group.jl +++ b/src/mdarray/group.jl @@ -253,7 +253,6 @@ function unsafe_createmdarray( group.dataset.value, ) end - function createmdarray( group::AbstractGroup, name::AbstractString, diff --git a/src/mdarray/types.jl b/src/mdarray/types.jl index 4e04a8ff..1ac75684 100644 --- a/src/mdarray/types.jl +++ b/src/mdarray/types.jl @@ -224,6 +224,7 @@ function destroy(dimension::AbstractDimension)::Nothing return nothing end +destroy(attributes::AbstractVector{<:AbstractAttribute}) = destroy.(attributes) destroy(dimensions::AbstractVector{<:AbstractDimension}) = destroy.(dimensions) ################################################################################ diff --git a/test/runtests.jl b/test/runtests.jl index 079c573b..b97365ef 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,8 +5,6 @@ using GDAL import ArchGDAL as AG import Aqua -include("test_mdarray.jl") - # ensure all testing files are present include("remotefiles.jl") diff --git a/test/test_mdarray.jl b/test/test_mdarray.jl index e1fee60a..5518849f 100644 --- a/test/test_mdarray.jl +++ b/test/test_mdarray.jl @@ -1,6 +1,8 @@ using Test import ArchGDAL as AG +# TODO: Test vsizip driver + # There should be more drivers... Anyone willing to update GDAL? # Possible drivers: at least HDF4, HDF5, TileDB const mdarray_drivers = [ From 2b353f320340e68cda44f6aaf232a10e8cdde86d Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 24 Jul 2024 14:50:07 -0400 Subject: [PATCH 11/19] CI: Disable benchmarks --- .github/workflows/benchmark.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 3f677d4f..8050a7d6 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -11,11 +11,11 @@ jobs: with: version: 1.6 - uses: julia-actions/julia-buildpkg@latest - - name: Install dependencies - run: julia -e 'using Pkg; pkg"add PkgBenchmark BenchmarkCI@0.1"' - - name: Run benchmarks - run: julia -e 'using BenchmarkCI; BenchmarkCI.judge()' - - name: Print judgement - run: julia -e 'using BenchmarkCI; BenchmarkCI.displayjudgement()' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + #TODO - name: Install dependencies + #TODO run: julia -e 'using Pkg; pkg"add PkgBenchmark BenchmarkCI@0.1"' + #TODO - name: Run benchmarks + #TODO run: julia -e 'using BenchmarkCI; BenchmarkCI.judge()' + #TODO - name: Print judgement + #TODO run: julia -e 'using BenchmarkCI; BenchmarkCI.displayjudgement()' + #TODO env: + #TODO GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From b57e23a2b4c8c4c223c340087cdeb60d7d040c00 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 25 Jul 2024 15:44:55 -0400 Subject: [PATCH 12/19] Test attributes --- src/ArchGDAL.jl | 3 +- src/context.jl | 3 + src/mdarray/attribute.jl | 615 ++++++++++++++++---------------- src/mdarray/extendeddatatype.jl | 155 ++++++++ src/mdarray/group.jl | 113 ++++++ src/mdarray/highlevel.jl | 69 ++++ src/mdarray/mdarray.jl | 170 +++++++-- src/mdarray/todo.jl | 13 - src/mdarray/types.jl | 4 +- test/test_mdarray.jl | 139 +++++++- 10 files changed, 927 insertions(+), 357 deletions(-) create mode 100644 src/mdarray/extendeddatatype.jl create mode 100644 src/mdarray/highlevel.jl delete mode 100644 src/mdarray/todo.jl diff --git a/src/ArchGDAL.jl b/src/ArchGDAL.jl index 4011c1ad..a10592b8 100644 --- a/src/ArchGDAL.jl +++ b/src/ArchGDAL.jl @@ -34,10 +34,11 @@ include("ogr/styletable.jl") include("mdarray/types.jl") include("mdarray/attribute.jl") include("mdarray/dimension.jl") +include("mdarray/extendeddatatype.jl") include("mdarray/global.jl") include("mdarray/group.jl") +include("mdarray/highlevel.jl") include("mdarray/mdarray.jl") -include("mdarray/todo.jl") include("utilities.jl") include("context.jl") include("base/iterators.jl") diff --git a/src/context.jl b/src/context.jl index 8a695e6b..98f9ed6f 100644 --- a/src/context.jl +++ b/src/context.jl @@ -221,6 +221,7 @@ for gdalfunc in ( :delaunaytriangulation, :difference, :extendeddatatypecreate, + :extendeddatatypecreatestring, :forceto, :fromGML, :fromJSON, @@ -238,6 +239,7 @@ for gdalfunc in ( :getattributes, :getband, :getcolortable, + :getcomponents, :getdatatype, :getdimensions, :getfeature, @@ -253,6 +255,7 @@ for gdalfunc in ( :getresampled, :getrootgroup, :getspatialref, + :gettype, :getunscaled, :getview, :importCRS, diff --git a/src/mdarray/attribute.jl b/src/mdarray/attribute.jl index c7195e7f..283d46ca 100644 --- a/src/mdarray/attribute.jl +++ b/src/mdarray/attribute.jl @@ -1,5 +1,25 @@ # GDALAttribute +const NumericAttributeType = Union{ + Int8, + Int16, + Int32, + Int64, + UInt8, + UInt16, + UInt32, + UInt64, + Float32, + Float64, + Complex{Int16}, + Complex{Int32}, + Complex{Float32}, + Complex{Float64}, +} +const ScalarAttributeType = Union{AbstractString,NumericAttributeType} +const AttributeType = + Union{ScalarAttributeType,AbstractVector{<:ScalarAttributeType}} + function getdimensionssize(attribute::AbstractAttribute)::NTuple{<:Any,Int} @assert !isnull(attribute) count = Ref{Csize_t}() @@ -18,77 +38,49 @@ function readasraw(attribute::AbstractAttribute)::AbstractVector{UInt8} return raw end -function readasstring(attribute::AbstractAttribute)::String - @assert !isnull(attribute) - stringptr = GDAL.gdalattributereadasstring(attribute) - string = unsafe_string(stringptr) - # do not free - return string -end - -function readasint(attribute::AbstractAttribute)::Int32 +function read(attribute::AbstractAttribute)::AttributeType @assert !isnull(attribute) - return GDAL.gdalattributereadasint(attribute) + rank = getdimensioncount(attribute) + length = gettotalelementscount(attribute) + rank == 0 && @assert length == 1 + datatype = getdatatype(attribute) + class = getclass(datatype) + + if class == GDAL.GEDTC_NUMERIC + # Read a numeric attribute + T = convert(DataType, getnumericdatatype(datatype)) + @assert T <: NumericAttributeType + count = Ref{Csize_t}() + ptr = GDAL.gdalattributereadasraw(attribute, count) + @assert count[] == length * sizeof(T) + if rank == 0 + # Read a scalar + value = unsafe_load(convert(Ptr{T}, ptr)) + GDAL.gdalattributefreerawresult(attribute, ptr, count[]) + return value + else + # Read a vector + values = T[unsafe_load(convert(Ptr{T}, ptr), n) for n in 1:length] + GDAL.gdalattributefreerawresult(attribute, ptr, count[]) + return values + end + + elseif class == GDAL.GEDTC_STRING + # Read a string attribute + if rank == 0 + return GDAL.gdalattributereadasstring(attribute) + else + return GDAL.gdalattributereadasstringarray(attribute) + end + + elseif class == GDAL.GEDTC_COMPOUND + # Read a compound attribute + error("unimplemented") + else + error("internal error") + end end -function readasint64(attribute::AbstractAttribute)::Int64 - @assert !isnull(attribute) - return GDAL.gdalattributereadasint64(attribute) -end - -function readasdouble(attribute::AbstractAttribute)::Float64 - @assert !isnull(attribute) - return GDAL.gdalattributereadasdouble(attribute) -end - -function readasstringarray(attribute::AbstractAttribute)::Vector{String} - @assert !isnull(attribute) - return GDAL.gdalattributereadasstringarray(attribute) -end - -function readasintarray(attribute::AbstractAttribute)::Vector{Int32} - @assert !isnull(attribute) - count = Ref{Csize_t}() - ptr = GDAL.gdalattributereadasintarray(attribute, count) - ptr == C_NULL && return Int32[] - values = Int32[unsafe_load(ptr, n) for n in 1:count[]] - GDAL.vsifree(ptr) - return values -end - -function readasint64array(attribute::AbstractAttribute)::Vector{Int64} - @assert !isnull(attribute) - count = Ref{Csize_t}() - ptr = GDAL.gdalattributereadasint64array(attribute, count) - ptr == C_NULL && return Int64[] - values = Int64[unsafe_load(ptr, n) for n in 1:count[]] - GDAL.vsifree(ptr) - return values -end - -function readasdoublearray(attribute::AbstractAttribute)::Vector{Float64} - @assert !isnull(attribute) - count = Ref{Csize_t}() - ptr = GDAL.gdalattributereadasdoublearray(attribute, count) - ptr == C_NULL && return Float64[] - values = Float64[unsafe_load(ptr, n) for n in 1:count[]] - GDAL.vsifree(ptr) - return values -end - -read(::Type{String}, attribute)::String = readasstring(attribute) -read(::Type{Int32}, attribute)::Int32 = readasint(attribute) -read(::Type{Int64}, attribute)::Int64 = readasint64(attribute) -read(::Type{Float64}, attribute)::Float64 = readasdouble(attribute) -read(::Type{Vector{String}}, attribute)::Vector{String} = - readasstringarray(attribute) -read(::Type{Vector{Int32}}, attribute)::Vector{Int32} = - readasintarray(attribute) -read(::Type{Vector{Int64}}, attribute)::Vector{Int64} = - readasint64array(attribute) -read(::Type{Vector{Float64}}, attribute)::Vector{Float64} = - readasdoublearray(attribute) - function writerraw( attribute::AbstractAttribute, value::AbstractVector{UInt8}, @@ -102,19 +94,21 @@ function write(attribute::AbstractAttribute, value::AbstractString)::Bool return Bool(GDAL.gdalattributewritestring(attribute, value)) end -function write(attribute::AbstractAttribute, value::Int32)::Bool - @assert !isnull(attribute) - return Bool(GDAL.gdalattributewriteint(attribute, value)) -end - -function write(attribute::AbstractAttribute, value::Int64)::Bool +function write(attribute::AbstractAttribute, value::NumericAttributeType)::Bool @assert !isnull(attribute) - return Bool(GDAL.gdalattributewriteint64(attribute, value)) -end - -function write(attribute::AbstractAttribute, value::Float64)::Bool - @assert !isnull(attribute) - return Bool(GDAL.gdalattributewritedouble(attribute, value)) + rank = getdimensioncount(attribute) + @assert rank == 0 + length = gettotalelementscount(attribute) + @assert length == 1 + datatype = getdatatype(attribute) + class = getclass(datatype) + @assert class == GDAL.GEDTC_NUMERIC + T = convert(DataType, getnumericdatatype(datatype)) + + valueT = convert(T, value)::T + return Bool( + GDAL.gdalattributewriteraw(attribute, Ref(valueT), sizeof(valueT)), + ) end function write( @@ -132,270 +126,269 @@ end function write( attribute::AbstractAttribute, - values::AbstractVector{Int32}, + values::AbstractVector{<:NumericAttributeType}, )::Bool @assert !isnull(attribute) - return Bool( - GDAL.gdalattributewriteintarray(attribute, values, length(values)), - ) -end + rank = getdimensioncount(attribute) + @assert rank == 1 + length = gettotalelementscount(attribute) + datatype = getdatatype(attribute) + class = getclass(datatype) + @assert class == GDAL.GEDTC_NUMERIC + T = convert(DataType, getnumericdatatype(datatype)) -function write( - attribute::AbstractAttribute, - values::AbstractVector{Int64}, -)::Bool - @assert !isnull(attribute) - return Bool( - GDAL.gdalattributewriteint64array(attribute, values, length(values)), - ) -end - -function write( - attribute::AbstractAttribute, - values::AbstractVector{Float64}, -)::Bool - @assert !isnull(attribute) - return Bool( - GDAL.gdalattributewritedoublearray(attribute, values, length(values)), - ) + valuesT = convert(Vector{T}, values)::Vector{T} + return Bool(GDAL.gdalattributewriteraw(attribute, valuesT, sizeof(valuesT))) end ################################################################################ -# For GDALGroup - -function unsafe_getattribute( - group::AbstractGroup, - name::AbstractString, -)::AbstractAttribute - @assert !isnull(group) - return Attribute( - GDAL.gdalgroupgetattribute(group, name), - group.dataset.value, - ) +function getname(attribute::AbstractAttribute)::AbstractString + @assert !isnull(attribute) + return GDAL.gdalattributegetname(attribute) end -function getattribute( - group::AbstractGroup, - name::AbstractString, -)::AbstractAttribute - @assert !isnull(group) - return IAttribute( - GDAL.gdalgroupgetattribute(group, name), - group.dataset.value, - ) +function getfullname(attribute::AbstractAttribute)::AbstractString + @assert !isnull(attribute) + return GDAL.gdalattributegetfullname(attribute) end -function unsafe_getattributes( - group::AbstractGroup, - options::OptionList = nothing, -)::AbstractVector{<:AbstractAttribute} - @assert !isnull(group) - count = Ref{Csize_t}() - ptr = - GDAL.gdalgroupgetattributes(group, count, CSLConstListWrapper(options)) - dataset = group.dataset.value - attributes = AbstractAttribute[ - Attribute(unsafe_load(ptr, n), dataset) for n in 1:count[] - ] - GDAL.vsifree(ptr) - return attributes +function gettotalelementscount(attribute::AbstractAttribute)::Int64 + @assert !isnull(attribute) + return Int64(GDAL.gdalattributegettotalelementscount(attribute)) end -function getattributes( - group::AbstractGroup, - options::OptionList = nothing, -)::AbstractVector{<:AbstractAttribute} - @assert !isnull(group) - count = Ref{Csize_t}() - ptr = - GDAL.gdalgroupgetattributes(group, count, CSLConstListWrapper(options)) - dataset = group.dataset.value - attributes = AbstractAttribute[ - IAttribute(unsafe_load(ptr, n), dataset) for n in 1:count[] - ] - GDAL.vsifree(ptr) - return attributes +function Base.length(attribute::AbstractAttribute) + @assert !isnull(attribute) + return Int(gettotalelementscount(attribute)) end -function unsafe_createattribute( - group::AbstractGroup, - name::AbstractString, - dimensions::AbstractVector{<:AbstractDimension}, - datatype::AbstractExtendedDataType, - options::OptionList = nothing, -)::AbstractAttribute - @assert !isnull(group) - @assert all(!isnull(dim) for dim in dimensions) - @assert !isnull(datatype) - return Attribute( - GDAL.gdalgroupcreateattribute( - group, - name, - length(dimensions), - DimensionHList(dimensions), - datatype, - CSLConstListWrapper(options), - ), - group.dataset.value, - ) +function getdimensioncount(attribute::AbstractAttribute)::Int + @assert !isnull(attribute) + return Int(GDAL.gdalattributegetdimensioncount(attribute)) end -function createattribute( - group::AbstractGroup, - name::AbstractString, - dimensions::AbstractVector{<:AbstractDimension}, - datatype::AbstractExtendedDataType, - options::OptionList = nothing, -)::AbstractAttribute - @assert !isnull(group) - @assert all(!isnull(dim) for dim in dimensions) - @assert !isnull(datatype) - return IAttribute( - GDAL.gdalgroupcreateattribute( - group, - name, - length(dimensions), - DimensionHList(dimensions), - datatype, - CSLConstListWrapper(options), - ), - group.dataset.value, - ) -end +Base.ndims(attribute::AbstractAttribute)::Int = getdimensioncount(attribute) -function deleteattribute( - group::AbstractGroup, - name::AbstractString, - options::OptionList = nothing, -)::Bool - @assert !isnull(group) - return GDAL.gdalgroupdeleteattribute( - group, - name, - CSLConstListWrapper(options), - ) +function getdimensions( + attribute::AbstractAttribute, +)::AbstractVector{<:AbstractDimension} + @assert !isnull(attribute) + dimensionscountref = Ref{Csize_t}() + dimensionshptr = + GDAL.gdalattributegetdimensions(attribute, dimensionscountref) + dataset = attribute.dataset.value + dimensions = AbstractDimension[ + IDimension(unsafe_load(dimensionshptr, n), dataset) for + n in 1:dimensionscountref[] + ] + GDAL.vsifree(dimensionshptr) + return dimensions end -################################################################################ - -# For GDALMDArray +function unsafe_getdimensions( + attribute::AbstractAttribute, +)::AbstractVector{<:AbstractDimension} + @assert !isnull(attribute) + dimensionscountref = Ref{Csize_t}() + dimensionshptr = + GDAL.gdalattributegetdimensions(attribute, dimensionscountref) + dataset = attribute.dataset.value + dimensions = AbstractDimension[ + Dimension(unsafe_load(dimensionshptr, n), dataset) for + n in 1:dimensionscountref[] + ] + GDAL.vsifree(dimensionshptr) + return dimensions +end -function unsafe_getattribute( - mdarray::AbstractMDArray, - name::AbstractString, -)::AbstractAttribute - @assert !isnull(mdarray) - return Attribute( - GDAL.gdalmdarraygetattribute(mdarray, name), - mdarray.dataset.value, - ) +function unsafe_getdatatype( + attribute::AbstractAttribute, +)::AbstractExtendedDataType + @assert !isnull(attribute) + return ExtendedDataType(GDAL.gdalattributegetdatatype(attribute)) end -function getattribute( - mdarray::AbstractMDArray, - name::AbstractString, -)::AbstractAttribute - @assert !isnull(mdarray) - return IAttribute( - GDAL.gdalmdarraygetattribute(mdarray, name), - mdarray.dataset.value, - ) +function getdatatype(attribute::AbstractAttribute)::AbstractExtendedDataType + @assert !isnull(attribute) + return IExtendedDataType(GDAL.gdalattributegetdatatype(attribute)) end -function unsafe_getattributes( - mdarray::AbstractMDArray, +function getblocksize( + attribute::AbstractAttribute, options::OptionList = nothing, -)::AbstractVector{<:AbstractAttribute} - @assert !isnull(mdarray) +)::AbstractVector{Int64} + @assert !isnull(attribute) count = Ref{Csize_t}() - ptr = GDAL.gdalmdarraygetattributes( - mdarray, + blocksizeptr = GDAL.gdalattributegetblocksize( + attribute, count, CSLConstListWrapper(options), ) - dataset = mdarray.dataset.value - attributes = AbstractAttribute[ - Attribute(unsafe_load(ptr, n), dataset) for n in 1:count[] - ] - GDAL.vsifree(ptr) - return attributes + blocksize = Int64[unsafe_load(blocksizeptr, n) for n in 1:count[]] + GDAL.vsifree(blocksizeptr) + return blocksize end -function getattributes( - mdarray::AbstractMDArray, - options::OptionList = nothing, -)::AbstractVector{<:AbstractAttribute} - @assert !isnull(mdarray) +function getprocessingchunksize( + attribute::AbstractAttribute, + maxchunkmemory::Integer, +)::AbstractVector{Int64} + @assert !isnull(attribute) count = Ref{Csize_t}() - ptr = GDAL.gdalmdarraygetattributes( - mdarray, + chunksizeptr = GDAL.gdalattributegetprocessingchunksize( + attribute, count, - CSLConstListWrapper(options), - ) - dataset = mdarray.dataset.value - attributes = AbstractAttribute[ - IAttribute(unsafe_load(ptr, n), dataset) for n in 1:count[] - ] - GDAL.vsifree(ptr) - return attributes -end - -function unsafe_createattribute( - mdarray::AbstractMDArray, - name::AbstractString, - dimensions::AbstractVector{<:AbstractDimension}, - datatype::AbstractExtendedDataType, - options::OptionList = nothing, -)::AbstractAttribute - @assert !isnull(mdarray) - @assert all(!isnull(dim) for dim in dimensions) - @assert !isnull(datatype) - return Attribute( - GDAL.gdalmdarraycreateattribute( - mdarray, - name, - length(dimensions), - DimensionHList(dimensions), - datatype, - CSLConstListWrapper(options), - ), - mdarray.dataset.value, - ) -end - -function createattribute( - mdarray::AbstractMDArray, - name::AbstractString, - dimensions::AbstractVector{<:AbstractDimension}, - datatype::AbstractExtendedDataType, - options::OptionList = nothing, -)::AbstractAttribute - @assert !isnull(mdarray) - @assert all(!isnull(dim) for dim in dimensions) - @assert !isnull(datatype) - return IAttribute( - GDAL.gdalmdarraycreateattribute( - mdarray, - name, - length(dimensions), - DimensionHList(dimensions), - datatype, - CSLConstListWrapper(options), - ), - mdarray.dataset.value, - ) -end - -function deleteattribute( - mdarray::AbstractMDArray, - name::AbstractString, - options::OptionList = nothing, -)::Bool - @assert !isnull(mdarray) - return GDAL.gdalmdarraydeleteattribute( - mdarray, - name, - CSLConstListWrapper(options), + maxchunkmemory, ) + chunksize = Int64[unsafe_load(chunksizeptr, n) for n in 1:count[]] + GDAL.vsifree(chunksizeptr) + return chunksize +end + +# processperchunk + +# function read!( +# attribute::AbstractAttribute, +# arraystartidx::IndexLike{D}, +# count::IndexLike{D}, +# arraystep::Union{Nothing,IndexLike{D}}, +# buffer::StridedArray{T,D}, +# )::Bool where {T,D} +# @assert !isnull(attribute) +# @assert length(arraystartidx) == D +# @assert length(count) == D +# @assert arraystep === nothing ? true : length(arraystep) == D +# gdal_arraystartidx = UInt64[arraystartidx[n] - 1 for n in D:-1:1] +# gdal_count = Csize_t[count[n] for n in D:-1:1] +# gdal_arraystep = +# arraystep === nothing ? nothing : Int64[arraystep[n] for n in D:-1:1] +# gdal_bufferstride = Cptrdiff_t[stride(buffer, n) for n in D:-1:1] +# return extendeddatatypecreate(T) do bufferdatatype +# return GDAL.gdalattributeread( +# attribute, +# gdal_arraystartidx, +# gdal_count, +# gdal_arraystep, +# gdal_bufferstride, +# bufferdatatype, +# buffer, +# buffer, +# sizeof(buffer), +# ) +# end +# end +# +# function read!( +# attribute::AbstractAttribute, +# region::RangeLike{D}, +# buffer::StridedArray{T,D}, +# )::Bool where {T,D} +# @assert length(region) == D +# arraystartidx = first.(region) +# count = length.(region) +# arraystep = step.(region) +# return read!(attribute, arraystartidx, count, arraystep, buffer) +# end +# +# function read!( +# attribute::AbstractAttribute, +# indices::CartesianIndices{D}, +# arraystep::Union{Nothing,IndexLike{D}}, +# buffer::StridedArray{T,D}, +# )::Bool where {T,D} +# @assert length(region) == D +# arraystartidx = first.(indices) +# count = length.(indices) +# return read!(attribute, arraystartidx, count, arraystep, buffer) +# end +# +# function read!( +# attribute::AbstractAttribute, +# indices::CartesianIndices{D}, +# buffer::StridedArray{T,D}, +# )::Bool where {T,D} +# return read!(attribute, indices, nothing, buffer) +# end +# +# function read!( +# attribute::AbstractAttribute, +# buffer::StridedArray{T,D}, +# )::Bool where {T,D} +# return read!(attribute, axes(buffer), buffer) +# end +# +# function write( +# attribute::AbstractAttribute, +# arraystartidx::IndexLike{D}, +# count::IndexLike{D}, +# arraystep::Union{Nothing,IndexLike{D}}, +# buffer::StridedArray{T,D}, +# )::Bool where {T,D} +# @assert !isnull(attribute) +# @assert length(arraystartidx) == D +# @assert length(count) == D +# @assert arraystep === nothing ? true : length(arraystep) == D +# gdal_arraystartidx = UInt64[arraystartidx[n] - 1 for n in D:-1:1] +# gdal_count = Csize_t[count[n] for n in D:-1:1] +# gdal_arraystep = +# arraystep === nothing ? nothing : Int64[arraystep[n] for n in D:-1:1] +# gdal_bufferstride = Cptrdiff_t[stride(buffer, n) for n in D:-1:1] +# return extendeddatatypecreate(T) do bufferdatatype +# return GDAL.gdalattributewrite( +# attribute, +# gdal_arraystartidx, +# gdal_count, +# gdal_arraystep, +# gdal_bufferstride, +# bufferdatatype, +# buffer, +# buffer, +# sizeof(buffer), +# ) +# end +# end +# +# function write( +# attribute::AbstractAttribute, +# region::RangeLike{D}, +# buffer::StridedArray{T,D}, +# )::Bool where {T,D} +# @assert length(region) == D +# arraystartidx = first.(region) +# count = length.(region) +# arraystep = step.(region) +# return write(attribute, arraystartidx, count, arraystep, buffer) +# end +# +# function write( +# attribute::AbstractAttribute, +# indices::CartesianIndices{D}, +# arraystep::Union{Nothing,IndexLike{D}}, +# buffer::StridedArray{T,D}, +# )::Bool where {T,D} +# @assert length(region) == D +# arraystartidx = first.(indices) +# count = length.(indices) +# return write(attribute, arraystartidx, count, arraystep, buffer) +# end +# +# function write( +# attribute::AbstractAttribute, +# indices::CartesianIndices{D}, +# buffer::StridedArray{T,D}, +# )::Bool where {T,D} +# return write(attribute, indices, nothing, buffer) +# end +# +# function write( +# attribute::AbstractAttribute, +# buffer::StridedArray{T,D}, +# )::Bool where {T,D} +# return write(attribute, axes(buffer), buffer) +# end + +function rename!(attribute::AbstractAttribute, newname::AbstractString)::Bool + @assert !isnull(attribute) + return GDAL.gdalattributerename(attribute, newname) end diff --git a/src/mdarray/extendeddatatype.jl b/src/mdarray/extendeddatatype.jl new file mode 100644 index 00000000..4dcbafcc --- /dev/null +++ b/src/mdarray/extendeddatatype.jl @@ -0,0 +1,155 @@ +# GDALExtendedDataType + +function Base.:(==)( + firstedt::AbstractExtendedDataType, + secondedt::AbstractExtendedDataType, +)::Bool + @assert !isnull(firstedt) + @assert !isnull(secondedt) + return Bool(GDAL.gdalextendeddatatypeequals(firstedt, secondedt)) +end + +function getname(edt::AbstractExtendedDataType)::AbstractString + @assert !isnull(edt) + return GDAL.gdalextendeddatatypegetname(edt) +end + +# TODO: Wrap GDAL.GDALExtendedDataTypeClass +function getclass(edt::AbstractExtendedDataType)::GDAL.GDALExtendedDataTypeClass + @assert !isnull(edt) + return GDAL.gdalextendeddatatypegetclass(edt) +end + +function getnumericdatatype(edt::AbstractExtendedDataType)::GDALDataType + @assert !isnull(edt) + return convert( + GDALDataType, + GDAL.gdalextendeddatatypegetnumericdatatype(edt), + ) +end + +# TODO: Wrap GDAL.GDALExtendedDataTypeSubType +function getsubtype( + edt::AbstractExtendedDataType, +)::GDAL.GDALExtendedDataTypeSubType + @assert !isnull(edt) + return GDAL.gdalextendeddatatypegetsubtype(edt) +end + +function unsafe_getcomponents( + edt::AbstractExtendedDataType, +)::AbstractVector{<:AbstractEDTComponent} + @assert !isnull(edt) + count = Ref{Csize_t}() + ptr = GDAL.gdalextendeddatatypegetcomponents(edt, count) + components = AbstractEDTComponent[ + EDTComponent(unsafe_load(ptr, n)) for n in 1:count[] + ] + GDAL.vsifree(ptr) + return components +end + +function getcomponents( + edt::AbstractExtendedDataType, +)::AbstractVector{<:AbstractEDTComponent} + @assert !isnull(edt) + count = Ref{Csize_t}() + ptr = GDAL.gdalextendeddatatypegetcomponents(edt, count) + components = AbstractEDTComponent[ + IEDTComponent(unsafe_load(ptr, n)) for n in 1:count[] + ] + GDAL.vsifree(ptr) + return components +end + +function getsize(edt::AbstractExtendedDataType)::Int + @assert !isnull(edt) + return Int(GDAL.gdalextendeddatatypegetsize(edt)) +end + +function getmaxstringlength(edt::AbstractExtendedDataType)::Int + @assert !isnull(edt) + return Int(GDAL.gdalextendeddatatypegetmaxstringlength(edt)) +end + +function canconvertto( + sourceedt::AbstractExtendedDataType, + targetedt::AbstractExtendedDataType, +)::Bool + @assert !isnull(sourceedt) + @assert !isnull(targetedt) + return Bool(GDAL.gdalextendeddatatypecanconvertto(sourceedt, targetedt)) +end + +# TODO: automate this +function needsfreedynamicmemory(edt::AbstractExtendedDataType)::Bool + return Bool(GDAL.gdalextendeddatatypeneedsfreedynamicmemory(edt)) +end + +function freedynamicmemory( + edt::AbstractExtendedDataType, + buffer::Ptr{Cvoid}, +)::Nothing + GDAL.gdalextendeddatatypefreedynamicmemory(edt, buffer) + return nothing +end + +################################################################################ + +function unsafe_extendeddatatypecreate( + ::Type{T}, +)::AbstractExtendedDataType where {T} + type = convert(GDALDataType, T) + return ExtendedDataType(GDAL.gdalextendeddatatypecreate(type)) +end + +function extendeddatatypecreate(::Type{T})::AbstractExtendedDataType where {T} + type = convert(GDALDataType, T) + return IExtendedDataType(GDAL.gdalextendeddatatypecreate(type)) +end + +# TODO: Wrap GDAL.GDALExtendedDataTypeSubType +function unsafe_extendeddatatypecreatestring( + maxstringlength::Integer = 0, + subtype::GDAL.GDALExtendedDataTypeSubType = GDAL.GEDTST_NONE, +)::AbstractExtendedDataType + return ExtendedDataType( + GDAL.gdalextendeddatatypecreatestringex(maxstringlength, subtype), + ) +end + +function extendeddatatypecreatestring( + maxstringlength::Integer = 0, + subtype::GDAL.GDALExtendedDataTypeSubType = GDAL.GEDTST_NONE, +)::AbstractExtendedDataType + return IExtendedDataType( + GDAL.gdalextendeddatatypecreatestringex(maxstringlength, subtype), + ) +end + +# copyvalue +# copyvalues + +################################################################################ + +# GDLEDTComponent + +function getname(comp::AbstractEDTComponent)::AbstractString + @assert !isnull(comp) + return GDAL.gdaledtcomponenttgetname(comp) +end + +function getoffset(comp::AbstractEDTComponent)::Int + @assert !isnull(comp) + return Int(GDAL.gdaledtcomponenttgetoffset(comp)) +end + +function unsafe_gettype(comp::AbstractEDTComponent)::AbstractExtendedDataType + @assert !isnull(comp) + return ExtendedDatatType(GDAL.gdaledtcomponenttgettype(comp)) +end + +function gettype(comp::AbstractEDTComponent)::AbstractExtendedDataType + @assert !isnull(comp) + return IExtendedDatatType(GDAL.gdaledtcomponenttgettype(comp)) +end diff --git a/src/mdarray/group.jl b/src/mdarray/group.jl index 98d4deac..0a33b19b 100644 --- a/src/mdarray/group.jl +++ b/src/mdarray/group.jl @@ -459,3 +459,116 @@ function subsetdimensionfromselection( group.dataset.value, ) end + +################################################################################ + +function unsafe_getattribute( + group::AbstractGroup, + name::AbstractString, +)::AbstractAttribute + @assert !isnull(group) + return Attribute( + GDAL.gdalgroupgetattribute(group, name), + group.dataset.value, + ) +end + +function getattribute( + group::AbstractGroup, + name::AbstractString, +)::AbstractAttribute + @assert !isnull(group) + return IAttribute( + GDAL.gdalgroupgetattribute(group, name), + group.dataset.value, + ) +end + +function unsafe_getattributes( + group::AbstractGroup, + options::OptionList = nothing, +)::AbstractVector{<:AbstractAttribute} + @assert !isnull(group) + count = Ref{Csize_t}() + ptr = + GDAL.gdalgroupgetattributes(group, count, CSLConstListWrapper(options)) + dataset = group.dataset.value + attributes = AbstractAttribute[ + Attribute(unsafe_load(ptr, n), dataset) for n in 1:count[] + ] + GDAL.vsifree(ptr) + return attributes +end + +function getattributes( + group::AbstractGroup, + options::OptionList = nothing, +)::AbstractVector{<:AbstractAttribute} + @assert !isnull(group) + count = Ref{Csize_t}() + ptr = + GDAL.gdalgroupgetattributes(group, count, CSLConstListWrapper(options)) + dataset = group.dataset.value + attributes = AbstractAttribute[ + IAttribute(unsafe_load(ptr, n), dataset) for n in 1:count[] + ] + GDAL.vsifree(ptr) + return attributes +end + +function unsafe_createattribute( + group::AbstractGroup, + name::AbstractString, + dimensions::AbstractVector{<:Integer}, + datatype::AbstractExtendedDataType, + options::OptionList = nothing, +)::AbstractAttribute + @assert !isnull(group) + @assert !isnull(datatype) + return Attribute( + GDAL.gdalgroupcreateattribute( + group, + name, + length(dimensions), + dimensions, + datatype, + CSLConstListWrapper(options), + ), + group.dataset.value, + ) +end + +function createattribute( + group::AbstractGroup, + name::AbstractString, + dimensions::AbstractVector{<:Integer}, + datatype::AbstractExtendedDataType, + options::OptionList = nothing, +)::AbstractAttribute + @assert !isnull(group) + @assert !isnull(datatype) + return IAttribute( + GDAL.gdalgroupcreateattribute( + group, + name, + length(dimensions), + dimensions, + datatype, + CSLConstListWrapper(options), + ), + group.dataset.value, + ) +end + +function deleteattribute( + group::AbstractGroup, + name::AbstractString, + options::OptionList = nothing, +)::Bool + @assert !isnull(group) + return GDAL.gdalgroupdeleteattribute( + group, + name, + CSLConstListWrapper(options), + ) +end diff --git a/src/mdarray/highlevel.jl b/src/mdarray/highlevel.jl new file mode 100644 index 00000000..c372a2bf --- /dev/null +++ b/src/mdarray/highlevel.jl @@ -0,0 +1,69 @@ +# High-level functions + +function writeattribute( + group::Union{AbstractGroup,AbstractMDArray}, + name::AbstractString, + value::AbstractString, +)::Bool + extendeddatatypecreatestring(length(value)) do datatype + createattribute(group, name, UInt64[], datatype) do attribute + return write(attribute, value) + end + end +end + +function writeattribute( + group::Union{AbstractGroup,AbstractMDArray}, + name::AbstractString, + value::NumericAttributeType, +)::Bool + extendeddatatypecreate(typeof(value)) do datatype + createattribute(group, name, UInt64[], datatype) do attribute + return write(attribute, value) + end + end +end + +function writeattribute( + group::Union{AbstractGroup,AbstractMDArray}, + name::AbstractString, + values::AbstractVector{<:AbstractString}, +)::Bool + extendeddatatypecreatestring(0) do datatype + createattribute( + group, + name, + UInt64[length(values)], + datatype, + ) do attribute + return write(attribute, values) + end + end +end + +function writeattribute( + group::Union{AbstractGroup,AbstractMDArray}, + name::AbstractString, + values::AbstractVector{<:NumericAttributeType}, +)::Bool + extendeddatatypecreate(eltype(values)) do datatype + createattribute( + group, + name, + UInt64[length(values)], + datatype, + ) do attribute + return write(attribute, values) + end + end +end + +function readattribute( + group::Union{AbstractGroup,AbstractMDArray}, + name::AbstractString, +)::Union{Nothing,AttributeType} + getattribute(group, name) do attribute + isnull(attribute) && return nothing + return read(attribute) + end +end diff --git a/src/mdarray/mdarray.jl b/src/mdarray/mdarray.jl index 16b82611..d4169520 100644 --- a/src/mdarray/mdarray.jl +++ b/src/mdarray/mdarray.jl @@ -594,38 +594,35 @@ end ################################################################################ -const AbstractAttributeOrMDArray = Union{AbstractAttribute,AbstractMDArray} - -function getname(mdarray::AbstractAttributeOrMDArray)::AbstractString +function getname(mdarray::AbstractMDArray)::AbstractString @assert !isnull(mdarray) return GDAL.gdalmdarraygetname(mdarray) end -function getfullname(mdarray::AbstractAttributeOrMDArray)::AbstractString +function getfullname(mdarray::AbstractMDArray)::AbstractString @assert !isnull(mdarray) return GDAL.gdalmdarraygetfullname(mdarray) end -function gettotalelementscount(mdarray::AbstractAttributeOrMDArray)::Int64 +function gettotalelementscount(mdarray::AbstractMDArray)::Int64 @assert !isnull(mdarray) return Int64(GDAL.gdalmdarraygettotalelementscount(mdarray)) end -function Base.length(mdarray::AbstractAttributeOrMDArray) +function Base.length(mdarray::AbstractMDArray) @assert !isnull(mdarray) return Int(gettotalelementscount(mdarray)) end -function getdimensioncount(mdarray::AbstractAttributeOrMDArray)::Int +function getdimensioncount(mdarray::AbstractMDArray)::Int @assert !isnull(mdarray) return Int(GDAL.gdalmdarraygetdimensioncount(mdarray)) end -Base.ndims(mdarray::AbstractAttributeOrMDArray)::Int = - getdimensioncount(mdarray) +Base.ndims(mdarray::AbstractMDArray)::Int = getdimensioncount(mdarray) function getdimensions( - mdarray::AbstractAttributeOrMDArray, + mdarray::AbstractMDArray, )::AbstractVector{<:AbstractDimension} @assert !isnull(mdarray) dimensionscountref = Ref{Csize_t}() @@ -640,7 +637,7 @@ function getdimensions( end function unsafe_getdimensions( - mdarray::AbstractAttributeOrMDArray, + mdarray::AbstractMDArray, )::AbstractVector{<:AbstractDimension} @assert !isnull(mdarray) dimensionscountref = Ref{Csize_t}() @@ -654,22 +651,18 @@ function unsafe_getdimensions( return dimensions end -function unsafe_getdatatype( - mdarray::AbstractAttributeOrMDArray, -)::AbstractExtendedDataType +function unsafe_getdatatype(mdarray::AbstractMDArray)::AbstractExtendedDataType @assert !isnull(mdarray) return ExtendedDataType(GDAL.gdalmdarraygetdatatype(mdarray)) end -function getdatatype( - mdarray::AbstractAttributeOrMDArray, -)::AbstractExtendedDataType +function getdatatype(mdarray::AbstractMDArray)::AbstractExtendedDataType @assert !isnull(mdarray) return IExtendedDataType(GDAL.gdalmdarraygetdatatype(mdarray)) end function getblocksize( - mdarray::AbstractAttributeOrMDArray, + mdarray::AbstractMDArray, options::OptionList = nothing, )::AbstractVector{Int64} @assert !isnull(mdarray) @@ -685,7 +678,7 @@ function getblocksize( end function getprocessingchunksize( - mdarray::AbstractAttributeOrMDArray, + mdarray::AbstractMDArray, maxchunkmemory::Integer, )::AbstractVector{Int64} @assert !isnull(mdarray) @@ -707,7 +700,7 @@ const RangeLike{D} = Union{ } function read!( - mdarray::AbstractAttributeOrMDArray, + mdarray::AbstractMDArray, arraystartidx::IndexLike{D}, count::IndexLike{D}, arraystep::Union{Nothing,IndexLike{D}}, @@ -738,7 +731,7 @@ function read!( end function read!( - mdarray::AbstractAttributeOrMDArray, + mdarray::AbstractMDArray, region::RangeLike{D}, buffer::StridedArray{T,D}, )::Bool where {T,D} @@ -750,7 +743,7 @@ function read!( end function read!( - mdarray::AbstractAttributeOrMDArray, + mdarray::AbstractMDArray, indices::CartesianIndices{D}, arraystep::Union{Nothing,IndexLike{D}}, buffer::StridedArray{T,D}, @@ -762,7 +755,7 @@ function read!( end function read!( - mdarray::AbstractAttributeOrMDArray, + mdarray::AbstractMDArray, indices::CartesianIndices{D}, buffer::StridedArray{T,D}, )::Bool where {T,D} @@ -770,14 +763,14 @@ function read!( end function read!( - mdarray::AbstractAttributeOrMDArray, + mdarray::AbstractMDArray, buffer::StridedArray{T,D}, )::Bool where {T,D} return read!(mdarray, axes(buffer), buffer) end function write( - mdarray::AbstractAttributeOrMDArray, + mdarray::AbstractMDArray, arraystartidx::IndexLike{D}, count::IndexLike{D}, arraystep::Union{Nothing,IndexLike{D}}, @@ -808,7 +801,7 @@ function write( end function write( - mdarray::AbstractAttributeOrMDArray, + mdarray::AbstractMDArray, region::RangeLike{D}, buffer::StridedArray{T,D}, )::Bool where {T,D} @@ -820,7 +813,7 @@ function write( end function write( - mdarray::AbstractAttributeOrMDArray, + mdarray::AbstractMDArray, indices::CartesianIndices{D}, arraystep::Union{Nothing,IndexLike{D}}, buffer::StridedArray{T,D}, @@ -832,7 +825,7 @@ function write( end function write( - mdarray::AbstractAttributeOrMDArray, + mdarray::AbstractMDArray, indices::CartesianIndices{D}, buffer::StridedArray{T,D}, )::Bool where {T,D} @@ -840,7 +833,7 @@ function write( end function write( - mdarray::AbstractAttributeOrMDArray, + mdarray::AbstractMDArray, buffer::StridedArray{T,D}, )::Bool where {T,D} return write(mdarray, axes(buffer), buffer) @@ -850,3 +843,122 @@ function rename!(mdarray::AbstractMDArray, newname::AbstractString)::Bool @assert !isnull(mdarray) return GDAL.gdalmdarrayrename(mdarray, newname) end + +################################################################################ + +function unsafe_getattribute( + mdarray::AbstractMDArray, + name::AbstractString, +)::AbstractAttribute + @assert !isnull(mdarray) + return Attribute( + GDAL.gdalmdarraygetattribute(mdarray, name), + mdarray.dataset.value, + ) +end + +function getattribute( + mdarray::AbstractMDArray, + name::AbstractString, +)::AbstractAttribute + @assert !isnull(mdarray) + return IAttribute( + GDAL.gdalmdarraygetattribute(mdarray, name), + mdarray.dataset.value, + ) +end + +function unsafe_getattributes( + mdarray::AbstractMDArray, + options::OptionList = nothing, +)::AbstractVector{<:AbstractAttribute} + @assert !isnull(mdarray) + count = Ref{Csize_t}() + ptr = GDAL.gdalmdarraygetattributes( + mdarray, + count, + CSLConstListWrapper(options), + ) + dataset = mdarray.dataset.value + attributes = AbstractAttribute[ + Attribute(unsafe_load(ptr, n), dataset) for n in 1:count[] + ] + GDAL.vsifree(ptr) + return attributes +end + +function getattributes( + mdarray::AbstractMDArray, + options::OptionList = nothing, +)::AbstractVector{<:AbstractAttribute} + @assert !isnull(mdarray) + count = Ref{Csize_t}() + ptr = GDAL.gdalmdarraygetattributes( + mdarray, + count, + CSLConstListWrapper(options), + ) + dataset = mdarray.dataset.value + attributes = AbstractAttribute[ + IAttribute(unsafe_load(ptr, n), dataset) for n in 1:count[] + ] + GDAL.vsifree(ptr) + return attributes +end + +function unsafe_createattribute( + mdarray::AbstractMDArray, + name::AbstractString, + dimensions::AbstractVector{<:Integer}, + datatype::AbstractExtendedDataType, + options::OptionList = nothing, +)::AbstractAttribute + @assert !isnull(mdarray) + @assert !isnull(datatype) + return Attribute( + GDAL.gdalmdarraycreateattribute( + mdarray, + name, + length(dimensions), + dimensions, + datatype, + CSLConstListWrapper(options), + ), + mdarray.dataset.value, + ) +end + +function createattribute( + mdarray::AbstractMDArray, + name::AbstractString, + dimensions::AbstractVector{<:Integer}, + datatype::AbstractExtendedDataType, + options::OptionList = nothing, +)::AbstractAttribute + @assert !isnull(mdarray) + @assert !isnull(datatype) + return IAttribute( + GDAL.gdalmdarraycreateattribute( + mdarray, + name, + length(dimensions), + dimensions, + datatype, + CSLConstListWrapper(options), + ), + mdarray.dataset.value, + ) +end + +function deleteattribute( + mdarray::AbstractMDArray, + name::AbstractString, + options::OptionList = nothing, +)::Bool + @assert !isnull(mdarray) + return GDAL.gdalmdarraydeleteattribute( + mdarray, + name, + CSLConstListWrapper(options), + ) +end diff --git a/src/mdarray/todo.jl b/src/mdarray/todo.jl deleted file mode 100644 index cd4e6301..00000000 --- a/src/mdarray/todo.jl +++ /dev/null @@ -1,13 +0,0 @@ -# GDALExtendedDataType - -function unsafe_extendeddatatypecreate( - ::Type{T}, -)::AbstractExtendedDataType where {T} - type = convert(GDALDataType, T) - return ExtendedDataType(GDAL.gdalextendeddatatypecreate(type)) -end - -function extendeddatatypecreate(::Type{T})::AbstractExtendedDataType where {T} - type = convert(GDALDataType, T) - return IExtendedDataType(GDAL.gdalextendeddatatypecreate(type)) -end diff --git a/src/mdarray/types.jl b/src/mdarray/types.jl index 1ac75684..ddb593aa 100644 --- a/src/mdarray/types.jl +++ b/src/mdarray/types.jl @@ -44,7 +44,6 @@ end mutable struct IEDTComponent <: AbstractEDTComponent ptr::GDAL.GDALEDTComponentH - dataset::WeakRef # AbstractDataset function IEDTComponent(ptr::GDAL.GDALEDTComponentH) edtcomponent = new(ptr) @@ -224,6 +223,9 @@ function destroy(dimension::AbstractDimension)::Nothing return nothing end +function destroy(edtcomponents::AbstractVector{<:AbstractEDTComponent}) + return destroy.(edtcomponents) +end destroy(attributes::AbstractVector{<:AbstractAttribute}) = destroy.(attributes) destroy(dimensions::AbstractVector{<:AbstractDimension}) = destroy.(dimensions) diff --git a/test/test_mdarray.jl b/test/test_mdarray.jl index 5518849f..cc1ca578 100644 --- a/test/test_mdarray.jl +++ b/test/test_mdarray.jl @@ -27,6 +27,119 @@ const mdarray_drivers = [ ), ] +# Attributes with complex values are not supported by Zarr (?) +const scalar_attribute_types = [ + String, + Int8, + Int16, + Int32, + Int64, + UInt8, + UInt16, + UInt32, + UInt64, + Float32, + Float64, + # Complex{Int16}, + # Complex{Int32}, + # Complex{Float32}, + # Complex{Float64}, +] +const attribute_types = + [scalar_attribute_types..., [Vector{T} for T in scalar_attribute_types]...] + +# Can't have `\0` or `/` in attribute names +# For netCDF: +# - first character must be alphanumeric or underscore or >= 128, +# - next characters cannot be iscontrol or DEL, +# - last character cannot be isspace +const attribute_names = [ + "attribute", + "αβγ", + # [string(ch) for ch in Char(1):Char(256) if ch != '/']..., + [ + string(ch) for ch in [ + ('A':'Z')..., + ('a':'z')..., + ('0':'9')..., + '_', + (Char(128):Char(256))..., + ] + ]..., +] + +get_attribute_value(::Type{String}) = "string" +get_attribute_value(::Type{T}) where {T<:Real} = T(32) +get_attribute_value(::Type{T}) where {T<:Complex} = T(32, 33) +get_attribute_value(::Type{Vector{String}}) = String["string", "", "αβγ"] +function get_attribute_value(::Type{Vector{T}}) where {T<:Integer} + # Can't store large Int64 values (JSON...) + tmin = + T == Int64 ? 1000 * T(typemin(Int32)) : + T == UInt64 ? 1000 * T(typemin(UInt32)) : typemin(T) + tmax = + T == Int64 ? 1000 * T(typemax(Int32)) : + T == UInt64 ? 1000 * T(typemax(UInt32)) : typemax(T) + return T[32, tmin, tmax, 0] +end +function get_attribute_value(::Type{Vector{T}}) where {T<:Real} + return T[ + 32, + typemin(T), + typemax(T), + T(+0.0), + T(-0.0), + eps(T), + prevfloat(T(1)), + nextfloat(T(1)), + T(Inf), + T(-Inf), + T(NaN), + ] +end +function get_attribute_value(::Type{Vector{T}}) where {T<:Complex{<:Integer}} + return T[T(32, 33), T(typemin(real(T)), typemax(real(T))), T(0)] +end +function get_attribute_value(::Type{Vector{T}}) where {T<:Complex{<:Real}} + return T[ + T(32, 33), + T(typemin(real(T)), typemax(real(T))), + T(0), + T(+0.0, -0.0), + T(eps(real(T))), + T(prevfloat(real(T)(1)), nextfloat(real(T)(1))), + T(Inf, -Inf), + T(NaN), + ] +end + +function write_attributes(loc::Union{AG.AbstractGroup,AG.AbstractMDArray}) + for name in attribute_names + @test AG.writeattribute(loc, name, name) + end + for T in attribute_types + @test AG.writeattribute(loc, "$T", get_attribute_value(T)) + end + return nothing +end +function test_attributes(loc::Union{AG.AbstractGroup,AG.AbstractMDArray}) + for name in attribute_names + @test isequal(AG.readattribute(loc, name), name) + if !isequal(AG.readattribute(loc, name), name) + @show loc name + @assert false + end + end + for T in attribute_types + @test isequal(AG.readattribute(loc, "$T"), get_attribute_value(T)) + if !isequal(AG.readattribute(loc, "$T"), get_attribute_value(T)) + @show loc T + @assert false + end + end + return nothing +end + @testset "test_mdarray.jl" begin @testset "$drivername" for ( drivername, @@ -72,6 +185,8 @@ const mdarray_drivers = [ @test AG.getgroupnames(root) == ["group"] @test AG.getgroupnames(group) == [] + write_attributes(group) + nx, ny = 3, 4 dimx = AG.createdimension(group, "x", "", "", nx) @test !AG.isnull(dimx) @@ -105,6 +220,8 @@ const mdarray_drivers = [ success = AG.write(mdarray, data) @test success + write_attributes(mdarray) + if drivername != "MEM" err = AG.close(dataset) @test err == GDAL.CE_None @@ -141,6 +258,8 @@ const mdarray_drivers = [ group = AG.opengroup(root, "group") @test !AG.isnull(group) + test_attributes(group) + mdarray = AG.openmdarray(group, "mdarray") @test !AG.isnull(mdarray) @@ -183,13 +302,16 @@ const mdarray_drivers = [ datatype = AG.getdatatype(mdarray) @test !AG.isnull(datatype) - # TODO: Check class + @test AG.getclass(datatype) == GDAL.GEDTC_NUMERIC + @test AG.getnumericdatatype(datatype) == AG.GDT_Float32 data = Array{Float32}(undef, ny, nx) success = AG.read!(mdarray, data) @test success @test data == Float32[100 * x + y for y in 1:ny, x in 1:nx] + test_attributes(mdarray) + err = AG.close(dataset) @test err == GDAL.CE_None @@ -229,6 +351,8 @@ const mdarray_drivers = [ @test AG.getgroupnames(root) == ["group"] @test AG.getgroupnames(group) == [] + write_attributes(group) + nx, ny = 3, 4 AG.createdimension(group, "x", "", "", nx) do dimx @test !AG.isnull(dimx) @@ -272,6 +396,9 @@ const mdarray_drivers = [ success = AG.write(mdarray, data) @test success + + write_attributes(mdarray) + return end end end @@ -305,6 +432,8 @@ const mdarray_drivers = [ AG.opengroup(root, "group") do group @test !AG.isnull(group) + test_attributes(group) + AG.openmdarray(group, "mdarray") do mdarray @test !AG.isnull(mdarray) @@ -333,7 +462,10 @@ const mdarray_drivers = [ datatype = AG.getdatatype(mdarray) @test !AG.isnull(datatype) - # TODO: Check class + @test AG.getclass(datatype) == + GDAL.GEDTC_NUMERIC + @test AG.getnumericdatatype(datatype) == + AG.GDT_Float32 AG.openmdarrayfromfullname( root, @@ -388,6 +520,9 @@ const mdarray_drivers = [ @test data == Float32[ 100 * x + y for y in 1:ny, x in 1:nx ] + + test_attributes(mdarray) + return end end end From 55b25b1e3cb03631966938ee959f5e23e3fdbfe8 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 25 Jul 2024 18:35:21 -0400 Subject: [PATCH 13/19] Test high-level mdarray read/write functions --- src/mdarray/attribute.jl | 140 --------------------------------------- src/mdarray/highlevel.jl | 28 ++++++++ src/mdarray/mdarray.jl | 16 +++++ test/test_mdarray.jl | 29 ++++++++ 4 files changed, 73 insertions(+), 140 deletions(-) diff --git a/src/mdarray/attribute.jl b/src/mdarray/attribute.jl index 283d46ca..4cdd8fde 100644 --- a/src/mdarray/attribute.jl +++ b/src/mdarray/attribute.jl @@ -248,146 +248,6 @@ end # processperchunk -# function read!( -# attribute::AbstractAttribute, -# arraystartidx::IndexLike{D}, -# count::IndexLike{D}, -# arraystep::Union{Nothing,IndexLike{D}}, -# buffer::StridedArray{T,D}, -# )::Bool where {T,D} -# @assert !isnull(attribute) -# @assert length(arraystartidx) == D -# @assert length(count) == D -# @assert arraystep === nothing ? true : length(arraystep) == D -# gdal_arraystartidx = UInt64[arraystartidx[n] - 1 for n in D:-1:1] -# gdal_count = Csize_t[count[n] for n in D:-1:1] -# gdal_arraystep = -# arraystep === nothing ? nothing : Int64[arraystep[n] for n in D:-1:1] -# gdal_bufferstride = Cptrdiff_t[stride(buffer, n) for n in D:-1:1] -# return extendeddatatypecreate(T) do bufferdatatype -# return GDAL.gdalattributeread( -# attribute, -# gdal_arraystartidx, -# gdal_count, -# gdal_arraystep, -# gdal_bufferstride, -# bufferdatatype, -# buffer, -# buffer, -# sizeof(buffer), -# ) -# end -# end -# -# function read!( -# attribute::AbstractAttribute, -# region::RangeLike{D}, -# buffer::StridedArray{T,D}, -# )::Bool where {T,D} -# @assert length(region) == D -# arraystartidx = first.(region) -# count = length.(region) -# arraystep = step.(region) -# return read!(attribute, arraystartidx, count, arraystep, buffer) -# end -# -# function read!( -# attribute::AbstractAttribute, -# indices::CartesianIndices{D}, -# arraystep::Union{Nothing,IndexLike{D}}, -# buffer::StridedArray{T,D}, -# )::Bool where {T,D} -# @assert length(region) == D -# arraystartidx = first.(indices) -# count = length.(indices) -# return read!(attribute, arraystartidx, count, arraystep, buffer) -# end -# -# function read!( -# attribute::AbstractAttribute, -# indices::CartesianIndices{D}, -# buffer::StridedArray{T,D}, -# )::Bool where {T,D} -# return read!(attribute, indices, nothing, buffer) -# end -# -# function read!( -# attribute::AbstractAttribute, -# buffer::StridedArray{T,D}, -# )::Bool where {T,D} -# return read!(attribute, axes(buffer), buffer) -# end -# -# function write( -# attribute::AbstractAttribute, -# arraystartidx::IndexLike{D}, -# count::IndexLike{D}, -# arraystep::Union{Nothing,IndexLike{D}}, -# buffer::StridedArray{T,D}, -# )::Bool where {T,D} -# @assert !isnull(attribute) -# @assert length(arraystartidx) == D -# @assert length(count) == D -# @assert arraystep === nothing ? true : length(arraystep) == D -# gdal_arraystartidx = UInt64[arraystartidx[n] - 1 for n in D:-1:1] -# gdal_count = Csize_t[count[n] for n in D:-1:1] -# gdal_arraystep = -# arraystep === nothing ? nothing : Int64[arraystep[n] for n in D:-1:1] -# gdal_bufferstride = Cptrdiff_t[stride(buffer, n) for n in D:-1:1] -# return extendeddatatypecreate(T) do bufferdatatype -# return GDAL.gdalattributewrite( -# attribute, -# gdal_arraystartidx, -# gdal_count, -# gdal_arraystep, -# gdal_bufferstride, -# bufferdatatype, -# buffer, -# buffer, -# sizeof(buffer), -# ) -# end -# end -# -# function write( -# attribute::AbstractAttribute, -# region::RangeLike{D}, -# buffer::StridedArray{T,D}, -# )::Bool where {T,D} -# @assert length(region) == D -# arraystartidx = first.(region) -# count = length.(region) -# arraystep = step.(region) -# return write(attribute, arraystartidx, count, arraystep, buffer) -# end -# -# function write( -# attribute::AbstractAttribute, -# indices::CartesianIndices{D}, -# arraystep::Union{Nothing,IndexLike{D}}, -# buffer::StridedArray{T,D}, -# )::Bool where {T,D} -# @assert length(region) == D -# arraystartidx = first.(indices) -# count = length.(indices) -# return write(attribute, arraystartidx, count, arraystep, buffer) -# end -# -# function write( -# attribute::AbstractAttribute, -# indices::CartesianIndices{D}, -# buffer::StridedArray{T,D}, -# )::Bool where {T,D} -# return write(attribute, indices, nothing, buffer) -# end -# -# function write( -# attribute::AbstractAttribute, -# buffer::StridedArray{T,D}, -# )::Bool where {T,D} -# return write(attribute, axes(buffer), buffer) -# end - function rename!(attribute::AbstractAttribute, newname::AbstractString)::Bool @assert !isnull(attribute) return GDAL.gdalattributerename(attribute, newname) diff --git a/src/mdarray/highlevel.jl b/src/mdarray/highlevel.jl index c372a2bf..2278ebb4 100644 --- a/src/mdarray/highlevel.jl +++ b/src/mdarray/highlevel.jl @@ -1,5 +1,33 @@ # High-level functions +function writemdarray( + group::AbstractGroup, + name::AbstractString, + value::StridedArray{T,D}, + options::OptionList = nothing, +)::Bool where {T<:NumericAttributeType,D} + dimensions = AbstractDimension[ + createdimension(group, "$name.$d", "", "", size(value, d)) for + d in D:-1:1 + ] + extendeddatatypecreate(T) do datatype + createmdarray(group, name, dimensions, datatype, options) do mdarray + return write(mdarray, value) + end + end +end + +function readmdarray( + group::AbstractGroup, + name::AbstractString, + options::OptionList = nothing, +)::Union{Nothing,AbstractArray} + openmdarray(group, name, options) do mdarray + isnull(mdarray) && return nothing + return read(mdarray) + end +end + function writeattribute( group::Union{AbstractGroup,AbstractMDArray}, name::AbstractString, diff --git a/src/mdarray/mdarray.jl b/src/mdarray/mdarray.jl index d4169520..7e2370f1 100644 --- a/src/mdarray/mdarray.jl +++ b/src/mdarray/mdarray.jl @@ -769,6 +769,22 @@ function read!( return read!(mdarray, axes(buffer), buffer) end +function read(mdarray::AbstractMDArray)::Union{Nothing,AbstractArray} + getdimensions(mdarray) do dimensions + D = length(dimensions) + sz = [getsize(dimensions[d]) for d in D:-1:1] + getdatatype(mdarray) do datatype + class = getclass(datatype) + @assert class == GDAL.GEDTC_NUMERIC + T = convert(DataType, getnumericdatatype(datatype)) + buffer = Array{T}(undef, sz...) + success = read!(mdarray, buffer) + !success && return nothing + return buffer + end + end +end + function write( mdarray::AbstractMDArray, arraystartidx::IndexLike{D}, diff --git a/test/test_mdarray.jl b/test/test_mdarray.jl index cc1ca578..5e53972a 100644 --- a/test/test_mdarray.jl +++ b/test/test_mdarray.jl @@ -222,6 +222,10 @@ end write_attributes(mdarray) + success = + AG.writemdarray(group, "primes", UInt8[2, 3, 5, 7, 251]) + @test success + if drivername != "MEM" err = AG.close(dataset) @test err == GDAL.CE_None @@ -310,8 +314,15 @@ end @test success @test data == Float32[100 * x + y for y in 1:ny, x in 1:nx] + data = AG.read(mdarray) + @test data !== nothing + @test data == Float32[100 * x + y for y in 1:ny, x in 1:nx] + test_attributes(mdarray) + primes = AG.readmdarray(group, "primes") + @test primes == UInt8[2, 3, 5, 7, 251] + err = AG.close(dataset) @test err == GDAL.CE_None @@ -398,6 +409,14 @@ end @test success write_attributes(mdarray) + + success = AG.writemdarray( + group, + "primes", + UInt8[2, 3, 5, 7, 251], + ) + @test success + return end end @@ -521,7 +540,17 @@ end 100 * x + y for y in 1:ny, x in 1:nx ] + data = AG.read(mdarray) + @test data !== nothing + @test data == Float32[ + 100 * x + y for y in 1:ny, x in 1:nx + ] + test_attributes(mdarray) + + primes = AG.readmdarray(group, "primes") + @test primes == UInt8[2, 3, 5, 7, 251] + return end end From 42685aa134beeb7a0ec18ade7267f3a7ef7a70cc Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 25 Jul 2024 18:41:40 -0400 Subject: [PATCH 14/19] Restore all test cases --- .github/workflows/benchmark.yml | 18 ++++----- test/runtests.jl | 70 ++++++++++++++++----------------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 8050a7d6..9e3c5078 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -4,18 +4,18 @@ on: [pull_request] jobs: Benchmark: - runs-on: macos-13 + runs-on: macos-latest steps: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@latest with: version: 1.6 - uses: julia-actions/julia-buildpkg@latest - #TODO - name: Install dependencies - #TODO run: julia -e 'using Pkg; pkg"add PkgBenchmark BenchmarkCI@0.1"' - #TODO - name: Run benchmarks - #TODO run: julia -e 'using BenchmarkCI; BenchmarkCI.judge()' - #TODO - name: Print judgement - #TODO run: julia -e 'using BenchmarkCI; BenchmarkCI.displayjudgement()' - #TODO env: - #TODO GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Install dependencies + run: julia -e 'using Pkg; pkg"add PkgBenchmark BenchmarkCI@0.1"' + - name: Run benchmarks + run: julia -e 'using BenchmarkCI; BenchmarkCI.judge()' + - name: Print judgement + run: julia -e 'using BenchmarkCI; BenchmarkCI.displayjudgement()' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/test/runtests.jl b/test/runtests.jl index b97365ef..608f1149 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -11,42 +11,42 @@ include("remotefiles.jl") @testset "ArchGDAL" begin cd(dirname(@__FILE__)) do isdir("tmp") || mkpath("tmp") - #TODO include("test_doctest.jl") - #TODO include("test_convert.jl") - #TODO include("test_tables.jl") - #TODO include("test_gdal_tutorials.jl") - #TODO include("test_geometry.jl") - #TODO include("test_types.jl") - #TODO include("test_display.jl") - #TODO include("test_drivers.jl") - #TODO include("test_feature.jl") - #TODO include("test_featurelayer.jl") - #TODO include("test_fielddefn.jl") - #TODO include("test_iterators.jl") - #TODO include("test_styletable.jl") - #TODO include("test_dataset.jl") - #TODO include("test_rasterband.jl") - #TODO include("test_rasterio.jl") - #TODO include("test_array.jl") - #TODO include("test_spatialref.jl") - #TODO include("test_gdalutilities.jl") - #TODO include("test_gdalutilities_errors.jl") - #TODO include("test_rasterattrtable.jl") + include("test_doctest.jl") + include("test_convert.jl") + include("test_tables.jl") + include("test_gdal_tutorials.jl") + include("test_geometry.jl") + include("test_types.jl") + include("test_display.jl") + include("test_drivers.jl") + include("test_feature.jl") + include("test_featurelayer.jl") + include("test_fielddefn.jl") + include("test_iterators.jl") + include("test_styletable.jl") + include("test_dataset.jl") + include("test_rasterband.jl") + include("test_rasterio.jl") + include("test_array.jl") + include("test_spatialref.jl") + include("test_gdalutilities.jl") + include("test_gdalutilities_errors.jl") + include("test_rasterattrtable.jl") include("test_mdarray.jl") - #TODO include("test_ospy_examples.jl") - #TODO include("test_geos_operations.jl") - #TODO include("test_cookbook_geometry.jl") - #TODO include("test_cookbook_projection.jl") - #TODO include("test_geotransform.jl") - #TODO include("test_images.jl") - #TODO include("test_utils.jl") - #TODO include("test_prepared_geometry.jl") - #TODO Aqua.test_all( - #TODO ArchGDAL; - #TODO ambiguities = false, - #TODO stale_deps = false, - #TODO piracies = false, - #TODO ) + include("test_ospy_examples.jl") + include("test_geos_operations.jl") + include("test_cookbook_geometry.jl") + include("test_cookbook_projection.jl") + include("test_geotransform.jl") + include("test_images.jl") + include("test_utils.jl") + include("test_prepared_geometry.jl") + Aqua.test_all( + ArchGDAL; + ambiguities = false, + stale_deps = false, + piracies = false, + ) return nothing end end From 2287f9f7152db2371cb82221571748100f0179c0 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 26 Jul 2024 12:10:04 -0400 Subject: [PATCH 15/19] Call error() when there are errors --- src/dataset.jl | 7 +- src/mdarray/dimension.jl | 20 +-- src/mdarray/global.jl | 4 +- src/mdarray/group.jl | 253 ++++++++++++++---------------- src/mdarray/highlevel.jl | 31 ++-- src/mdarray/mdarray.jl | 327 +++++++++++++++++++++++---------------- src/mdarray/types.jl | 30 ++-- src/types.jl | 20 +-- test/runtests.jl | 2 +- test/test_mdarray.jl | 213 +++++++++++++------------ 10 files changed, 478 insertions(+), 429 deletions(-) diff --git a/src/dataset.jl b/src/dataset.jl index ca41764f..5a391112 100644 --- a/src/dataset.jl +++ b/src/dataset.jl @@ -1016,8 +1016,11 @@ end # TODO: Wrap `GDAL.CPLErr` function close(dataset::AbstractDataset)::GDAL.CPLErr dataset.ptr == C_NULL && return GDAL.CE_Failure - if dataset.children !== nothing - destroy.(dataset.children) + if !isnothing(dataset.children) + for child in dataset.children + value = child.value + !isnothing(value) && destroy(value) + end Base.empty!(dataset.children) end err = GDAL.gdalclose(dataset) diff --git a/src/mdarray/dimension.jl b/src/mdarray/dimension.jl index 544a4401..5e7ef0d7 100644 --- a/src/mdarray/dimension.jl +++ b/src/mdarray/dimension.jl @@ -29,26 +29,26 @@ function unsafe_getindexingvariable( dimension::AbstractDimension, )::AbstractMDArray @assert !isnull(dimension) - return MDArray( - GDAL.gdaldimensiongetindexingvariable(dimension), - dimension.dataset.value, - ) + ptr = GDAL.gdaldimensiongetindexingvariable(dimension) + ptr == C_NULL && error("Could not get indexing variable for dimension") + return MDArray(ptr, dimension.dataset.value) end function getindexingvariable(dimension::AbstractDimension)::AbstractMDArray @assert !isnull(dimension) - return IMDArray( - GDAL.gdaldimensiongetindexingvariable(dimension), - dimension.dataset.value, - ) + ptr = GDAL.gdaldimensiongetindexingvariable(dimension) + ptr == C_NULL && error("Could not get indexing variable for dimension") + return IMDArray(ptr, dimension.dataset.value) end function setindexingvariable!( dimension::AbstractDimension, indexingvariable::AbstractMDArray, -)::Bool +)::Nothing @assert !isnull(dimension) - return GDAL.gdaldimensionsetindexingvariable(dimension, indexingvariable) + success = GDAL.gdaldimensionsetindexingvariable(dimension, indexingvariable) + success == 0 && error("Could not set indexing variable for dimension") + return nothing end function rename!(dimension::AbstractDimension, newname::AbstractString)::Bool diff --git a/src/mdarray/global.jl b/src/mdarray/global.jl index e79dc484..248e433c 100644 --- a/src/mdarray/global.jl +++ b/src/mdarray/global.jl @@ -48,7 +48,7 @@ function unsafe_open( siblingfiles::OptionList, hard_close::Union{Nothing,Bool} = nothing, )::AbstractDataset - if hard_close === nothing + if isnothing(hard_close) # We hard-close the dataset if it is a writable multidim dataset hard_close = (openflags & OF_MULTIDIM_RASTER != 0) && @@ -74,7 +74,7 @@ function open( siblingfiles::OptionList, hard_close::Union{Nothing,Bool} = nothing, )::AbstractDataset - if hard_close === nothing + if isnothing(hard_close) # We hard-close the dataset if it is a writable multidim dataset hard_close = (openflags & OF_MULTIDIM_RASTER != 0) && diff --git a/src/mdarray/group.jl b/src/mdarray/group.jl index 0a33b19b..0011a567 100644 --- a/src/mdarray/group.jl +++ b/src/mdarray/group.jl @@ -24,10 +24,9 @@ function unsafe_openmdarray( options::OptionList = nothing, )::AbstractMDArray @assert !isnull(group) - return MDArray( - GDAL.gdalgroupopenmdarray(group, name, CSLConstListWrapper(options)), - group.dataset.value, - ) + ptr = GDAL.gdalgroupopenmdarray(group, name, CSLConstListWrapper(options)) + ptr == C_NULL && error("Could not open mdarray \"$name\"") + return MDArray(ptr, group.dataset.value) end function openmdarray( @@ -36,10 +35,9 @@ function openmdarray( options::OptionList = nothing, )::AbstractMDArray @assert !isnull(group) - return IMDArray( - GDAL.gdalgroupopenmdarray(group, name, CSLConstListWrapper(options)), - group.dataset.value, - ) + ptr = GDAL.gdalgroupopenmdarray(group, name, CSLConstListWrapper(options)) + ptr == C_NULL && error("Could not open mdarray \"$name\"") + return IMDArray(ptr, group.dataset.value) end function getgroupnames( @@ -56,10 +54,9 @@ function unsafe_opengroup( options::OptionList = nothing, )::AbstractGroup @assert !isnull(group) - return Group( - GDAL.gdalgroupopengroup(group, name, CSLConstListWrapper(options)), - group.dataset.value, - ) + ptr = GDAL.gdalgroupopengroup(group, name, CSLConstListWrapper(options)) + ptr == C_NULL && error("Could no open group \"$name\"") + return Group(ptr, group.dataset.value) end function opengroup( @@ -68,10 +65,9 @@ function opengroup( options::OptionList = nothing, )::AbstractGroup @assert !isnull(group) - return IGroup( - GDAL.gdalgroupopengroup(group, name, CSLConstListWrapper(options)), - group.dataset.value, - ) + ptr = GDAL.gdalgroupopengroup(group, name, CSLConstListWrapper(options)) + ptr == C_NULL && error("Could no open group \"$name\"") + return IGroup(ptr, group.dataset.value) end function getvectorlayernames( @@ -159,10 +155,9 @@ function unsafe_creategroup( options::OptionList = nothing, )::AbstractGroup @assert !isnull(group) - return Group( - GDAL.gdalgroupcreategroup(group, name, CSLConstListWrapper(options)), - group.dataset.value, - ) + ptr = GDAL.gdalgroupcreategroup(group, name, CSLConstListWrapper(options)) + ptr == C_NULL && error("Could not create group \"$name\"") + return Group(ptr, group.dataset.value) end function creategroup( @@ -171,10 +166,9 @@ function creategroup( options::OptionList = nothing, )::AbstractGroup @assert !isnull(group) - return IGroup( - GDAL.gdalgroupcreategroup(group, name, CSLConstListWrapper(options)), - group.dataset.value, - ) + ptr = GDAL.gdalgroupcreategroup(group, name, CSLConstListWrapper(options)) + ptr == C_NULL && error("Could not create group \"$name\"") + return IGroup(ptr, group.dataset.value) end function deletegroup( @@ -196,17 +190,16 @@ function unsafe_createdimension( options::OptionList = nothing, )::AbstractDimension @assert !isnull(group) - return Dimension( - GDAL.gdalgroupcreatedimension( - group, - name, - type, - direction, - size, - CSLConstListWrapper(options), - ), - group.dataset.value, + ptr = GDAL.gdalgroupcreatedimension( + group, + name, + type, + direction, + size, + CSLConstListWrapper(options), ) + ptr == C_NULL && error("Could not create dimension \"$name\"") + return Dimension(ptr, group.dataset.value) end function createdimension( @@ -218,17 +211,16 @@ function createdimension( options::OptionList = nothing, )::AbstractDimension @assert !isnull(group) - return IDimension( - GDAL.gdalgroupcreatedimension( - group, - name, - type, - direction, - size, - CSLConstListWrapper(options), - ), - group.dataset.value, + ptr = GDAL.gdalgroupcreatedimension( + group, + name, + type, + direction, + size, + CSLConstListWrapper(options), ) + ptr == C_NULL && error("Could not create dimension \"$name\"") + return IDimension(ptr, group.dataset.value) end function unsafe_createmdarray( @@ -241,18 +233,18 @@ function unsafe_createmdarray( @assert !isnull(group) @assert all(!isnull(dim) for dim in dimensions) @assert !isnull(datatype) - return MDArray( - GDAL.gdalgroupcreatemdarray( - group, - name, - length(dimensions), - DimensionHList(dimensions), - datatype, - CSLConstListWrapper(options), - ), - group.dataset.value, + ptr = GDAL.gdalgroupcreatemdarray( + group, + name, + length(dimensions), + DimensionHList(dimensions), + datatype, + CSLConstListWrapper(options), ) + ptr == C_NULL && error("Could not create mdarray \"$name\"") + return MDArray(ptr, group.dataset.value) end + function createmdarray( group::AbstractGroup, name::AbstractString, @@ -263,17 +255,16 @@ function createmdarray( @assert !isnull(group) @assert all(!isnull(dim) for dim in dimensions) @assert !isnull(datatype) - return IMDArray( - GDAL.gdalgroupcreatemdarray( - group, - name, - length(dimensions), - DimensionHList(dimensions), - datatype, - CSLConstListWrapper(options), - ), - group.dataset.value, + ptr = GDAL.gdalgroupcreatemdarray( + group, + name, + length(dimensions), + DimensionHList(dimensions), + datatype, + CSLConstListWrapper(options), ) + ptr == C_NULL && error("Could not create mdarray \"$name\"") + return IMDArray(ptr, group.dataset.value) end function deletemdarray( @@ -305,14 +296,13 @@ function unsafe_openmdarrayfromfullname( options::OptionList = nothing, )::AbstractMDArray @assert !isnull(group) - return MDArray( - GDAL.gdalgroupopenmdarrayfromfullname( - group, - fullname, - CSLConstListWrapper(options), - ), - group.dataset.value, + ptr = GDAL.gdalgroupopenmdarrayfromfullname( + group, + fullname, + CSLConstListWrapper(options), ) + ptr == C_NULL && error("Could not open mdarray \"$fullname\"") + return MDArray(ptr, group.dataset.value) end function openmdarrayfromfullname( @@ -321,14 +311,13 @@ function openmdarrayfromfullname( options::OptionList = nothing, )::AbstractMDArray @assert !isnull(group) - return IMDArray( - GDAL.gdalgroupopenmdarrayfromfullname( - group, - fullname, - CSLConstListWrapper(options), - ), - group.dataset.value, + ptr = GDAL.gdalgroupopenmdarrayfromfullname( + group, + fullname, + CSLConstListWrapper(options), ) + ptr == C_NULL && error("Could not open mdarray \"$fullname\"") + return IMDArray(ptr, group.dataset.value) end function unsafe_resolvemdarray( @@ -338,15 +327,14 @@ function unsafe_resolvemdarray( options::OptionList = nothing, )::AbstractMDArray @assert !isnull(group) - return MDArray( - GDAL.gdalgroupresolvemdarray( - group, - name, - startingpath, - CSLConstListWrapper(options), - ), - group.dataset.value, + ptr = GDAL.gdalgroupresolvemdarray( + group, + name, + startingpath, + CSLConstListWrapper(options), ) + ptr == C_NULL && error("Could not resolve mdarray \"$name\"") + return MDArray(ptr, group.dataset.value) end function resolvemdarray( @@ -356,15 +344,14 @@ function resolvemdarray( options::OptionList = nothing, )::AbstractMDArray @assert !isnull(group) - return IMDArray( - GDAL.gdalgroupresolvemdarray( - group, - name, - startingpath, - CSLConstListWrapper(options), - ), - group.dataset.value, + ptr = GDAL.gdalgroupresolvemdarray( + group, + name, + startingpath, + CSLConstListWrapper(options), ) + ptr == C_NULL && error("Could not resolve mdarray \"$name\"") + return IMDArray(ptr, group.dataset.value) end function unsafe_opengroupfromfullname( @@ -373,14 +360,13 @@ function unsafe_opengroupfromfullname( options::OptionList = nothing, )::AbstractGroup @assert !isnull(group) - return Group( - GDAL.gdalgroupopengroupfromfullname( - group, - fullname, - CSLConstListWrapper(options), - ), - group.dataset.value, + ptr = GDAL.gdalgroupopengroupfromfullname( + group, + fullname, + CSLConstListWrapper(options), ) + ptr == C_NULL && error("Could not open group \"$fullname\"") + return Group(ptr, group.dataset.value) end function opengroupfromfullname( @@ -389,14 +375,13 @@ function opengroupfromfullname( options::OptionList = nothing, )::AbstractGroup @assert !isnull(group) - return IGroup( - GDAL.gdalgroupopengroupfromfullname( - group, - fullname, - CSLConstListWrapper(options), - ), - group.dataset.value, + ptr = GDAL.gdalgroupopengroupfromfullname( + group, + fullname, + CSLConstListWrapper(options), ) + ptr == C_NULL && error("Could not open group \"$fullname\"") + return IGroup(ptr, group.dataset.value) end # function unsafe_opendimensionfromfullname( @@ -467,10 +452,9 @@ function unsafe_getattribute( name::AbstractString, )::AbstractAttribute @assert !isnull(group) - return Attribute( - GDAL.gdalgroupgetattribute(group, name), - group.dataset.value, - ) + ptr = GDAL.gdalgroupgetattribute(group, name) + ptr == C_NULL && error("Could not open attribute \"$name\"") + return Attribute(ptr, group.dataset.value) end function getattribute( @@ -478,10 +462,9 @@ function getattribute( name::AbstractString, )::AbstractAttribute @assert !isnull(group) - return IAttribute( - GDAL.gdalgroupgetattribute(group, name), - group.dataset.value, - ) + ptr = GDAL.gdalgroupgetattribute(group, name) + ptr == C_NULL && error("Could not open attribute \"$name\"") + return IAttribute(ptr, group.dataset.value) end function unsafe_getattributes( @@ -525,17 +508,16 @@ function unsafe_createattribute( )::AbstractAttribute @assert !isnull(group) @assert !isnull(datatype) - return Attribute( - GDAL.gdalgroupcreateattribute( - group, - name, - length(dimensions), - dimensions, - datatype, - CSLConstListWrapper(options), - ), - group.dataset.value, + ptr = GDAL.gdalgroupcreateattribute( + group, + name, + length(dimensions), + dimensions, + datatype, + CSLConstListWrapper(options), ) + ptr == C_NULL && error("Could not create attribute \"$name\"") + return Attribute(ptr, group.dataset.value) end function createattribute( @@ -547,17 +529,16 @@ function createattribute( )::AbstractAttribute @assert !isnull(group) @assert !isnull(datatype) - return IAttribute( - GDAL.gdalgroupcreateattribute( - group, - name, - length(dimensions), - dimensions, - datatype, - CSLConstListWrapper(options), - ), - group.dataset.value, + ptr = GDAL.gdalgroupcreateattribute( + group, + name, + length(dimensions), + dimensions, + datatype, + CSLConstListWrapper(options), ) + ptr == C_NULL && error("Could not create attribute \"$name\"") + return IAttribute(ptr, group.dataset.value) end function deleteattribute( diff --git a/src/mdarray/highlevel.jl b/src/mdarray/highlevel.jl index 2278ebb4..eb7be715 100644 --- a/src/mdarray/highlevel.jl +++ b/src/mdarray/highlevel.jl @@ -5,14 +5,15 @@ function writemdarray( name::AbstractString, value::StridedArray{T,D}, options::OptionList = nothing, -)::Bool where {T<:NumericAttributeType,D} +)::Nothing where {T<:NumericAttributeType,D} dimensions = AbstractDimension[ createdimension(group, "$name.$d", "", "", size(value, d)) for d in D:-1:1 ] extendeddatatypecreate(T) do datatype createmdarray(group, name, dimensions, datatype, options) do mdarray - return write(mdarray, value) + write(mdarray, value) + return nothing end end end @@ -21,9 +22,8 @@ function readmdarray( group::AbstractGroup, name::AbstractString, options::OptionList = nothing, -)::Union{Nothing,AbstractArray} +)::AbstractArray openmdarray(group, name, options) do mdarray - isnull(mdarray) && return nothing return read(mdarray) end end @@ -32,10 +32,11 @@ function writeattribute( group::Union{AbstractGroup,AbstractMDArray}, name::AbstractString, value::AbstractString, -)::Bool +)::Nothing extendeddatatypecreatestring(length(value)) do datatype createattribute(group, name, UInt64[], datatype) do attribute - return write(attribute, value) + write(attribute, value) + return nothing end end end @@ -44,10 +45,11 @@ function writeattribute( group::Union{AbstractGroup,AbstractMDArray}, name::AbstractString, value::NumericAttributeType, -)::Bool +)::Nothing extendeddatatypecreate(typeof(value)) do datatype createattribute(group, name, UInt64[], datatype) do attribute - return write(attribute, value) + write(attribute, value) + return nothing end end end @@ -56,7 +58,7 @@ function writeattribute( group::Union{AbstractGroup,AbstractMDArray}, name::AbstractString, values::AbstractVector{<:AbstractString}, -)::Bool +)::Nothing extendeddatatypecreatestring(0) do datatype createattribute( group, @@ -64,7 +66,8 @@ function writeattribute( UInt64[length(values)], datatype, ) do attribute - return write(attribute, values) + write(attribute, values) + return nothing end end end @@ -73,7 +76,7 @@ function writeattribute( group::Union{AbstractGroup,AbstractMDArray}, name::AbstractString, values::AbstractVector{<:NumericAttributeType}, -)::Bool +)::Nothing extendeddatatypecreate(eltype(values)) do datatype createattribute( group, @@ -81,7 +84,8 @@ function writeattribute( UInt64[length(values)], datatype, ) do attribute - return write(attribute, values) + write(attribute, values) + return nothing end end end @@ -89,9 +93,8 @@ end function readattribute( group::Union{AbstractGroup,AbstractMDArray}, name::AbstractString, -)::Union{Nothing,AttributeType} +)::AttributeType getattribute(group, name) do attribute - isnull(attribute) && return nothing return read(attribute) end end diff --git a/src/mdarray/mdarray.jl b/src/mdarray/mdarray.jl index 7e2370f1..999ee208 100644 --- a/src/mdarray/mdarray.jl +++ b/src/mdarray/mdarray.jl @@ -1,5 +1,25 @@ # GDALMDArray +function MDArray(ptr::GDAL.GDALMDArrayH, dataset::AbstractDataset) + @assert ptr != C_NULL + datatype = IExtendedDataType(GDAL.gdalmdarraygetdatatype(ptr)) + class = getclass(datatype) + @assert class == GDAL.GEDTC_NUMERIC + T = convert(DataType, getnumericdatatype(datatype)) + D = Int(GDAL.gdalmdarraygetdimensioncount(ptr)) + return MDArray{T,D}(ptr, dataset) +end + +function IMDArray(ptr::GDAL.GDALMDArrayH, dataset::AbstractDataset) + @assert ptr != C_NULL + datatype = IExtendedDataType(GDAL.gdalmdarraygetdatatype(ptr)) + class = getclass(datatype) + @assert class == GDAL.GEDTC_NUMERIC + T = convert(DataType, getnumericdatatype(datatype)) + D = Int(GDAL.gdalmdarraygetdimensioncount(ptr)) + return IMDArray{T,D}(ptr, dataset) +end + # function iswritable(mdarray::AbstractMDArray)::Bool # return GDAL.gdalmdarrayiswritable(mdarray) # end @@ -172,7 +192,7 @@ function setoffset!( return GDAL.gdalmdarraysetoffset( mdarray, offset, - storagetype === nothing ? GDAL.GDT_Unknown : + isnothing(storagetype) ? GDAL.GDT_Unknown : convert(GDAL.GDALDataType, storagetype), ) end @@ -186,7 +206,7 @@ function setscale!( return GDAL.gdalmdarraysetscale( mdarray, offset, - storagetype === nothing ? GDAL.GDT_Unknown : + isnothing(storagetype) ? GDAL.GDT_Unknown : convert(GDAL.GDALDataType, storagetype), ) end @@ -196,10 +216,9 @@ function unsafe_getview( viewexpr::AbstractString, )::AbstractMDArray @assert !isnull(mdarray) - return MDArray( - GDAL.gdalmdarraygetview(mdarray, viewexpr), - mdarray.dataset.value, - ) + ptr = GDAL.gdalmdarraygetview(mdarray, viewexpr) + ptr == C_NULL && error("Could not get view \"$vierexpr\"") + return MDArray(ptr, mdarray.dataset.value) end function getview( @@ -207,10 +226,9 @@ function getview( viewexpr::AbstractString, )::AbstractMDArray @assert !isnull(mdarray) - return IMDArray( - GDAL.gdalmdarraygetview(mdarray, viewexpr), - mdarray.dataset.value, - ) + ptr = GDAL.gdalmdarraygetview(mdarray, viewexpr) + ptr == C_NULL && error("Could not get view \"$vierexpr\"") + return IMDArray(ptr, mdarray.dataset.value) end function unsafe_getindex( @@ -252,12 +270,16 @@ end # TODO: Return a `LinearAlgebra.Adjoint` instead? function unsafe_transpose(mdarray::AbstractMDArray)::AbstractMDArray @assert !isnull(mdarray) - return MDArray(GDAL.gdalmdarraytranspose(mdarray), mdarray.dataset.value) + ptr = GDAL.gdalmdarraytranspose(mdarray) + ptr == C_NULL && error("Could not transpose mdarray") + return MDArray(ptr, mdarray.dataset.value) end function transpose(mdarray::AbstractMDArray)::AbstractMDArray @assert !isnull(mdarray) - return IMDArray(GDAL.gdalmdarraytranspose(mdarray), mdarray.dataset.value) + ptr = GDAL.gdalmdarraytranspose(mdarray) + ptr == C_NULL && error("Could not transpose mdarray") + return IMDArray(ptr, mdarray.dataset.value) end function unsafe_getunscaled( @@ -267,15 +289,14 @@ function unsafe_getunscaled( overriddendstnodata = Float64(NaN), )::AbstractMDArray @assert !isnull(mdarray) - return MDArray( - GDAL.gdalmdarraygetunscaled( - mdarray, - overriddenscale, - overriddenoffset, - overriddendstnodata, - ), - mdarray.dataset.value, + ptr = GDAL.gdalmdarraygetunscaled( + mdarray, + overriddenscale, + overriddenoffset, + overriddendstnodata, ) + ptr == C_NULL && error("Could not get unscaled mdarray") + return MDArray(ptr, mdarray.dataset.value) end function getunscaled( @@ -285,15 +306,14 @@ function getunscaled( overriddendstnodata = Float64(NaN), )::AbstractMDArray @assert !isnull(mdarray) - return IMDArray( - GDAL.gdalmdarraygetunscaled( - mdarray, - overriddenscale, - overriddenoffset, - overriddendstnodata, - ), - mdarray.dataset.value, + ptr = GDAL.gdalmdarraygetunscaled( + mdarray, + overriddenscale, + overriddenoffset, + overriddendstnodata, ) + ptr == C_NULL && error("Could not get unscaled mdarray") + return IMDArray(ptr, mdarray.dataset.value) end function unsafe_getmask( @@ -301,10 +321,9 @@ function unsafe_getmask( options::OptionList = nothing, )::AbstractMDArray @assert !isnull(mdarray) - return MDArray( - GDAL.gdalmdarraygetmask(mdarray, CSLConstListWrapper(options)), - mdarray.dataset.value, - ) + ptr = GDAL.gdalmdarraygetmask(mdarray, CSLConstListWrapper(options)) + ptr == C_NULL && error("Could not get mask for mdarray") + return MDArray(ptr, mdarray.dataset.value) end function getmask( @@ -312,10 +331,9 @@ function getmask( options::OptionList = nothing, )::AbstractMDArray @assert !isnull(mdarray) - return IMDArray( - GDAL.gdalmdarraygetmask(mdarray, CSLConstListWrapper(options)), - mdarray.dataset.value, - ) + ptr = GDAL.gdalmdarraygetmask(mdarray, CSLConstListWrapper(options)) + ptr == C_NULL && error("Could not get mask for mdarray") + return IMDArray(ptr, mdarray.dataset.value) end # TODO: Wrap GDAL.GDALRIOResampleAlg @@ -327,17 +345,16 @@ function unsafe_getresampled( options::OptionList = nothing, )::AbstractMDArray @assert !isnull(mdarray) - return MDArray( - GDAL.gdalmdarraygetresampled( - mdarray, - newdims === nothing ? 0 : length(newdims), - newdims === nothing ? C_NULL : DimensionHList(newdims), - resamplealg, - targetsrs == nothing ? C_NULL : targetsrs, - CSLConstListWrapper(options), - ), - mdarray.dataset.value, + ptr = GDAL.gdalmdarraygetresampled( + mdarray, + isnothing(newdims) ? 0 : length(newdims), + isnothing(newdims) ? C_NULL : DimensionHList(newdims), + resamplealg, + isnothing(targetsrs) ? C_NULL : targetsrs, + CSLConstListWrapper(options), ) + ptr == C_NULL && error("Could not get resampled mdarray") + return MDArray(ptr, mdarray.dataset.value) end function getresampled( @@ -348,17 +365,16 @@ function getresampled( options::OptionList = nothing, )::AbstractMDArray @assert !isnull(mdarray) - return IMDArray( - GDAL.gdalmdarraygetresampled( - mdarray, - newdims === nothing ? 0 : length(newdims), - newdims === nothing ? C_NULL : DimensionHList(newdims), - resamplealg, - targetsrs == nothing ? C_NULL : targetsrs, - CSLConstListWrapper(options), - ), - mdarray.dataset.value, + ptr = GDAL.gdalmdarraygetresampled( + mdarray, + isnothing(newdims) ? 0 : length(newdims), + isnothing(newdims) ? C_NULL : DimensionHList(newdims), + resamplealg, + isnothing(targetsrs) ? C_NULL : targetsrs, + CSLConstListWrapper(options), ) + ptr == C_NULL && error("Could not get resampled mdarray") + return IMDArray(ptr, mdarray.dataset.value) end function unsafe_getgridded( @@ -371,16 +387,15 @@ function unsafe_getgridded( @assert !isnull(mdarray) @assert !isnull(xarray) @assert !isnull(yarray) - return MDArray( - GDAL.gdalmdarraygetgridded( - mdarray, - gridoptions, - xarray, - yarray, - CSLConstListWrapper(options), - ), - mdarray.dataset.value, + ptr = GDAL.gdalmdarraygetgridded( + mdarray, + gridoptions, + xarray, + yarray, + CSLConstListWrapper(options), ) + ptr == C_NULL && error("Could not get gridded mdarray") + return MDArray(ptr, mdarray.dataset.value) end function getgridded( @@ -393,16 +408,15 @@ function getgridded( @assert !isnull(mdarray) @assert !isnull(xarray) @assert !isnull(yarray) - return IMDArray( - GDAL.gdalmdarraygetgridded( - mdarray, - gridoptions, - xarray, - yarray, - CSLConstListWrapper(options), - ), - mdarray.dataset.value, + ptr = GDAL.gdalmdarraygetgridded( + mdarray, + gridoptions, + xarray, + yarray, + CSLConstListWrapper(options), ) + ptr == C_NULL && error("Could not get gridded mdarray") + return IMDArray(ptr, mdarray.dataset.value) end function unsafe_asclassicdataset( @@ -413,13 +427,13 @@ function unsafe_asclassicdataset( options::OptionList = nothing, )::AbstractDataset @assert !isnull(mdarray) - @assert rootgroup === nothing || !isnull(rootgroup) + @assert isnothing(rootgroup) || !isnull(rootgroup) return Dataset( GDAL.gdalmdarrayasclassicdataset( mdarray, xdim, ydim, - rootgroup === nothing ? C_NULL : rootgroup, + isnothing(rootgroup) ? C_NULL : rootgroup, CSLConstListWrapper(options), ), ) @@ -433,13 +447,13 @@ function asclassicdataset( options::OptionList = nothing, )::AbstractDataset @assert !isnull(mdarray) - @assert rootgroup === nothing || !isnull(rootgroup) + @assert isnothing(rootgroup) || !isnull(rootgroup) return IDataset( GDAL.gdalmdarrayasclassicdataset( mdarray, xdim, ydim, - rootgroup === nothing ? C_NULL : rootgroup, + isnothing(rootgroup) ? C_NULL : rootgroup, CSLConstListWrapper(options), ), ) @@ -447,14 +461,18 @@ end function unsafe_asmdarray(rasterband::AbstractRasterBand)::AbstractMDArray @assert !isnull(rasterband) + ptr = GADL.gdalrasterbandasmdarray(rasterband) + ptr == C_NULL && error("Could not get view rasterband view as mdarray") # TODO: Find dataset - return MDArray(GADL.gdalrasterbandasmdarray(rasterband)) + return MDArray(ptr) end function asmdarray(rasterband::AbstractRasterBand)::AbstractMDArray @assert !isnull(rasterband) + ptr = GADL.gdalrasterbandasmdarray(rasterband) + ptr == C_NULL && error("Could not get view rasterband view as mdarray") # TODO: Find dataset - return IMDArray(GADL.gdalrasterbandasmdarray(rasterband)) + return IMDArray(ptr) end # TODO: Wrap GDAL.CPLErr @@ -513,7 +531,7 @@ function computestatistics( C_NULL, CSLConstListWrapper(options), ) - return succeess, min[], max[], mean[], stddev[], Int64(validcount[]) + return Bool(succeess), min[], max[], mean[], stddev[], Int64(validcount[]) end function clearstatistics(mdarray::AbstractMDArray)::Nothing @@ -546,8 +564,8 @@ function adviseread( @assert !isnull(mdarray) return GDAL.gdalmdarrayadviseread( mdarray, - arraystartidx === nothing ? C_NULL : arraystartidx, - count === nothing ? C_NULL : count, + isnothing(arraystartidx) ? C_NULL : arraystartidx, + isnothing(count) ? C_NULL : count, CSLConstListWrapper(options), ) end @@ -614,10 +632,11 @@ function Base.length(mdarray::AbstractMDArray) return Int(gettotalelementscount(mdarray)) end -function getdimensioncount(mdarray::AbstractMDArray)::Int - @assert !isnull(mdarray) - return Int(GDAL.gdalmdarraygetdimensioncount(mdarray)) -end +# function getdimensioncount(mdarray::AbstractMDArray)::Int +# @assert !isnull(mdarray) +# return Int(GDAL.gdalmdarraygetdimensioncount(mdarray)) +# end +getdimensioncount(mdarray::AbstractMDArray{<:Any,D}) where {D} = D Base.ndims(mdarray::AbstractMDArray)::Int = getdimensioncount(mdarray) @@ -651,6 +670,13 @@ function unsafe_getdimensions( return dimensions end +function Base.size(mdarray::AbstractMDArray) + getdimensions(mdarray) do dimensions + D = length(dimensions) + return ntuple(d -> getsize(dimensions[D-d+1]), D) + end +end + function unsafe_getdatatype(mdarray::AbstractMDArray)::AbstractExtendedDataType @assert !isnull(mdarray) return ExtendedDataType(GDAL.gdalmdarraygetdatatype(mdarray)) @@ -661,6 +687,8 @@ function getdatatype(mdarray::AbstractMDArray)::AbstractExtendedDataType return IExtendedDataType(GDAL.gdalmdarraygetdatatype(mdarray)) end +Base.eltype(mdarray::AbstractMDArray{T}) where {T} = T + function getblocksize( mdarray::AbstractMDArray, options::OptionList = nothing, @@ -672,11 +700,19 @@ function getblocksize( count, CSLConstListWrapper(options), ) - blocksize = Int64[unsafe_load(blocksizeptr, n) for n in 1:count[]] + blocksize = Int64[unsafe_load(blocksizeptr, n) for n in count[]:-1:1] GDAL.vsifree(blocksizeptr) return blocksize end +DiskArrays.haschunks(::AbstractMDArray) = DiskArrays.Chunked() +function DiskArrays.eachchunk( + mdarray::AbstractMDArray{<:Any,D}, +)::NTuple{D,Int} where {D} + blocksize = getblocksize(mdarray) + return DiskArrays.GridChunks(mdarray, Int.(blocksize)) +end + function getprocessingchunksize( mdarray::AbstractMDArray, maxchunkmemory::Integer, @@ -685,7 +721,7 @@ function getprocessingchunksize( count = Ref{Csize_t}() chunksizeptr = GDAL.gdalmdarraygetprocessingchunksize(mdarray, count, maxchunkmemory) - chunksize = Int64[unsafe_load(chunksizeptr, n) for n in 1:count[]] + chunksize = Int64[unsafe_load(chunksizeptr, n) for n in count[]:-1:1] GDAL.vsifree(chunksizeptr) return chunksize end @@ -705,18 +741,18 @@ function read!( count::IndexLike{D}, arraystep::Union{Nothing,IndexLike{D}}, buffer::StridedArray{T,D}, -)::Bool where {T,D} +)::Nothing where {T,D} @assert !isnull(mdarray) @assert length(arraystartidx) == D @assert length(count) == D - @assert arraystep === nothing ? true : length(arraystep) == D + @assert isnothing(arraystep) ? true : length(arraystep) == D gdal_arraystartidx = UInt64[arraystartidx[n] - 1 for n in D:-1:1] gdal_count = Csize_t[count[n] for n in D:-1:1] gdal_arraystep = - arraystep === nothing ? nothing : Int64[arraystep[n] for n in D:-1:1] + isnothing(arraystep) ? nothing : Int64[arraystep[n] for n in D:-1:1] gdal_bufferstride = Cptrdiff_t[stride(buffer, n) for n in D:-1:1] return extendeddatatypecreate(T) do bufferdatatype - return GDAL.gdalmdarrayread( + success = GDAL.gdalmdarrayread( mdarray, gdal_arraystartidx, gdal_count, @@ -727,6 +763,8 @@ function read!( buffer, sizeof(buffer), ) + success == 0 && error("Could not read mdarray") + return nothing end end @@ -734,7 +772,7 @@ function read!( mdarray::AbstractMDArray, region::RangeLike{D}, buffer::StridedArray{T,D}, -)::Bool where {T,D} +)::Nothing where {T,D} @assert length(region) == D arraystartidx = first.(region) count = length.(region) @@ -747,7 +785,7 @@ function read!( indices::CartesianIndices{D}, arraystep::Union{Nothing,IndexLike{D}}, buffer::StridedArray{T,D}, -)::Bool where {T,D} +)::Nothing where {T,D} @assert length(region) == D arraystartidx = first.(indices) count = length.(indices) @@ -758,18 +796,18 @@ function read!( mdarray::AbstractMDArray, indices::CartesianIndices{D}, buffer::StridedArray{T,D}, -)::Bool where {T,D} +)::Nothing where {T,D} return read!(mdarray, indices, nothing, buffer) end function read!( mdarray::AbstractMDArray, buffer::StridedArray{T,D}, -)::Bool where {T,D} +)::Nothing where {T,D} return read!(mdarray, axes(buffer), buffer) end -function read(mdarray::AbstractMDArray)::Union{Nothing,AbstractArray} +function read(mdarray::AbstractMDArray)::AbstractArray getdimensions(mdarray) do dimensions D = length(dimensions) sz = [getsize(dimensions[d]) for d in D:-1:1] @@ -778,31 +816,40 @@ function read(mdarray::AbstractMDArray)::Union{Nothing,AbstractArray} @assert class == GDAL.GEDTC_NUMERIC T = convert(DataType, getnumericdatatype(datatype)) buffer = Array{T}(undef, sz...) - success = read!(mdarray, buffer) - !success && return nothing + read!(mdarray, buffer) return buffer end end end +function DiskArrays.readblock!( + mdarray::AbstractMDArray{<:Any,D}, + aout, + r::Vararg{AbstractUnitRange,D}, +)::Nothing where {D} + success == read!(mdarray, r, aout) + @assert success + return nothing +end + function write( mdarray::AbstractMDArray, arraystartidx::IndexLike{D}, count::IndexLike{D}, arraystep::Union{Nothing,IndexLike{D}}, buffer::StridedArray{T,D}, -)::Bool where {T,D} +)::Nothing where {T,D} @assert !isnull(mdarray) @assert length(arraystartidx) == D @assert length(count) == D - @assert arraystep === nothing ? true : length(arraystep) == D + @assert isnothing(arraystep) ? true : length(arraystep) == D gdal_arraystartidx = UInt64[arraystartidx[n] - 1 for n in D:-1:1] gdal_count = Csize_t[count[n] for n in D:-1:1] gdal_arraystep = - arraystep === nothing ? nothing : Int64[arraystep[n] for n in D:-1:1] + isnothing(arraystep) ? nothing : Int64[arraystep[n] for n in D:-1:1] gdal_bufferstride = Cptrdiff_t[stride(buffer, n) for n in D:-1:1] return extendeddatatypecreate(T) do bufferdatatype - return GDAL.gdalmdarraywrite( + success = GDAL.gdalmdarraywrite( mdarray, gdal_arraystartidx, gdal_count, @@ -813,6 +860,8 @@ function write( buffer, sizeof(buffer), ) + success == 0 && error("Could not write mdarray") + return nothing end end @@ -820,7 +869,7 @@ function write( mdarray::AbstractMDArray, region::RangeLike{D}, buffer::StridedArray{T,D}, -)::Bool where {T,D} +)::Nothing where {T,D} @assert length(region) == D arraystartidx = first.(region) count = length.(region) @@ -833,7 +882,7 @@ function write( indices::CartesianIndices{D}, arraystep::Union{Nothing,IndexLike{D}}, buffer::StridedArray{T,D}, -)::Bool where {T,D} +)::Nothing where {T,D} @assert length(region) == D arraystartidx = first.(indices) count = length.(indices) @@ -844,17 +893,27 @@ function write( mdarray::AbstractMDArray, indices::CartesianIndices{D}, buffer::StridedArray{T,D}, -)::Bool where {T,D} +)::Nothing where {T,D} return write(mdarray, indices, nothing, buffer) end function write( mdarray::AbstractMDArray, buffer::StridedArray{T,D}, -)::Bool where {T,D} +)::Nothing where {T,D} return write(mdarray, axes(buffer), buffer) end +function DiskArrays.writeblock!( + mdarray::AbstractMDArray{<:Any,D}, + ain, + r::Vararg{AbstractUnitRange,D}, +)::Nothing where {D} + success == write(mdarray, r, ain) + @assert success + return nothing +end + function rename!(mdarray::AbstractMDArray, newname::AbstractString)::Bool @assert !isnull(mdarray) return GDAL.gdalmdarrayrename(mdarray, newname) @@ -867,10 +926,9 @@ function unsafe_getattribute( name::AbstractString, )::AbstractAttribute @assert !isnull(mdarray) - return Attribute( - GDAL.gdalmdarraygetattribute(mdarray, name), - mdarray.dataset.value, - ) + ptr = GDAL.gdalmdarraygetattribute(mdarray, name) + ptr == C_NULL && error("Could not get attribute \"$name\"") + return Attribute(ptr, mdarray.dataset.value) end function getattribute( @@ -878,10 +936,9 @@ function getattribute( name::AbstractString, )::AbstractAttribute @assert !isnull(mdarray) - return IAttribute( - GDAL.gdalmdarraygetattribute(mdarray, name), - mdarray.dataset.value, - ) + ptr = GDAL.gdalmdarraygetattribute(mdarray, name) + ptr == C_NULL && error("Could not get attribute \"$name\"") + return IAttribute(ptr, mdarray.dataset.value) end function unsafe_getattributes( @@ -931,17 +988,16 @@ function unsafe_createattribute( )::AbstractAttribute @assert !isnull(mdarray) @assert !isnull(datatype) - return Attribute( - GDAL.gdalmdarraycreateattribute( - mdarray, - name, - length(dimensions), - dimensions, - datatype, - CSLConstListWrapper(options), - ), - mdarray.dataset.value, + ptr = GDAL.gdalmdarraycreateattribute( + mdarray, + name, + length(dimensions), + dimensions, + datatype, + CSLConstListWrapper(options), ) + ptr == C_NULL && error("Could not create attribute \"$name\"") + return Attribute(ptr, mdarray.dataset.value) end function createattribute( @@ -953,17 +1009,16 @@ function createattribute( )::AbstractAttribute @assert !isnull(mdarray) @assert !isnull(datatype) - return IAttribute( - GDAL.gdalmdarraycreateattribute( - mdarray, - name, - length(dimensions), - dimensions, - datatype, - CSLConstListWrapper(options), - ), - mdarray.dataset.value, + ptr = GDAL.gdalmdarraycreateattribute( + mdarray, + name, + length(dimensions), + dimensions, + datatype, + CSLConstListWrapper(options), ) + ptr == C_NULL && error("Could not create attribute \"$name\"") + return IAttribute(ptr, mdarray.dataset.value) end function deleteattribute( diff --git a/src/mdarray/types.jl b/src/mdarray/types.jl index ddb593aa..715167e5 100644 --- a/src/mdarray/types.jl +++ b/src/mdarray/types.jl @@ -4,14 +4,14 @@ abstract type AbstractExtendedDataType end abstract type AbstractEDTComponent end # needs to have a `ptr::GDALEDTComponentH` attribute +# TODO: <: AbstractDict abstract type AbstractGroup end # needs to have a `ptr::GDAL.GDALGroupH` attribute -# TODO: Add `<: AbstractDiskArray` -# TODO: Put `{T,D}` into the type signature? -abstract type AbstractMDArray end +abstract type AbstractMDArray{T,D} <: AbstractDiskArray{T,D} end # needs to have a `ptr::GDAL.GDALMDArrayH` attribute +# TODO: <: DenseArray{T,D} abstract type AbstractAttribute end # needs to have a `ptr::GDAL.GDALAttributeH` attribute @@ -75,23 +75,33 @@ mutable struct IGroup <: AbstractGroup end end -mutable struct MDArray <: AbstractMDArray +mutable struct MDArray{T,D} <: AbstractMDArray{T,D} ptr::GDAL.GDALMDArrayH dataset::WeakRef # AbstractDataset - function MDArray(ptr::GDAL.GDALMDArrayH, dataset::AbstractDataset) - mdarray = new(ptr, WeakRef(dataset)) + function MDArray{T,D}( + ptr::GDAL.GDALMDArrayH, + dataset::AbstractDataset, + ) where {T,D} + T::Type + D::Int + mdarray = new{T,D}(ptr, WeakRef(dataset)) add_child!(dataset, mdarray) return mdarray end end -mutable struct IMDArray <: AbstractMDArray +mutable struct IMDArray{T,D} <: AbstractMDArray{T,D} ptr::GDAL.GDALMDArrayH dataset::WeakRef # AbstractDataset - function IMDArray(ptr::GDAL.GDALMDArrayH, dataset::AbstractDataset) - mdarray = new(ptr, WeakRef(dataset)) + function IMDArray{T,D}( + ptr::GDAL.GDALMDArrayH, + dataset::AbstractDataset, + ) where {T,D} + T::Type + D::Int + mdarray = new{T,D}(ptr, WeakRef(dataset)) add_child!(dataset, mdarray) ptr != C_NULL && finalizer(destroy, mdarray) return mdarray @@ -258,7 +268,7 @@ function CSLConstListWrapper(strings::AbstractVector{<:AbstractString}) end function Base.cconvert(::Type{GDAL.CSLConstList}, wrapper::CSLConstListWrapper) - wrapper.cstrings === nothing && + isnothing(wrapper.cstrings) && return Base.cconvert(GDAL.CSLConstList, C_NULL) return Base.cconvert(GDAL.CSLConstList, wrapper.cstrings) end diff --git a/src/types.jl b/src/types.jl index 34eb2ef4..1af67143 100644 --- a/src/types.jl +++ b/src/types.jl @@ -48,19 +48,19 @@ end # Each child must have a `destroy` function. mutable struct Dataset <: AbstractDataset ptr::GDAL.GDALDatasetH - children::Union{Nothing,Vector{Any}} + children::Union{Nothing,Vector{WeakRef}} function Dataset(ptr::GDAL.GDALDatasetH = C_NULL; hard_close::Bool = false) - return new(ptr, hard_close ? [] : nothing) + return new(ptr, hard_close ? WeakRef[] : nothing) end end mutable struct IDataset <: AbstractDataset ptr::GDAL.GDALDatasetH - children::Union{Nothing,Vector{Any}} + children::Union{Nothing,Vector{WeakRef}} function IDataset(ptr::GDAL.GDALDatasetH = C_NULL; hard_close::Bool = false) - dataset = new(ptr, hard_close ? [] : nothing) + dataset = new(ptr, hard_close ? WeakRef[] : nothing) finalizer(destroy, dataset) return dataset end @@ -69,8 +69,8 @@ end function add_child!(dataset::AbstractDataset, obj::Any)::Nothing isnull(obj) && return nothing @assert !isnull(dataset) - dataset.children === nothing && return nothing - push!(dataset.children, obj) + isnothing(dataset.children) && return nothing + push!(dataset.children, WeakRef(obj)) return nothing end @@ -640,11 +640,11 @@ end for T in (GDALOpenFlag, FieldValidation) eval(quote - Base.:&(x::$T, y::UInt32)::UInt32 = UInt32(x) & y - Base.:&(x::UInt32, y::$T)::UInt32 = x & UInt32(y) + Base.:&(x::$T, y::Unsigned)::UInt32 = UInt32(x) & UInt32(y) + Base.:&(x::Unsigned, y::$T)::UInt32 = UInt32(x) & UInt32(y) Base.:&(x::$T, y::$T)::UInt32 = UInt32(x) & UInt32(y) - Base.:|(x::$T, y::UInt32)::UInt32 = UInt32(x) | y - Base.:|(x::UInt32, y::$T)::UInt32 = x | UInt32(y) + Base.:|(x::$T, y::Unsigned)::UInt32 = UInt32(x) | UInt32(y) + Base.:|(x::Unsigned, y::$T)::UInt32 = UInt32(x) | UInt32(y) Base.:|(x::$T, y::$T)::UInt32 = UInt32(x) | UInt32(y) end) end diff --git a/test/runtests.jl b/test/runtests.jl index 608f1149..addfd68f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3,7 +3,7 @@ using Dates using GDAL #TODO import ArchGDAL import ArchGDAL as AG -import Aqua +using Aqua # ensure all testing files are present include("remotefiles.jl") diff --git a/test/test_mdarray.jl b/test/test_mdarray.jl index 5e53972a..85b29a17 100644 --- a/test/test_mdarray.jl +++ b/test/test_mdarray.jl @@ -1,5 +1,6 @@ using Test import ArchGDAL as AG +using GDAL # TODO: Test vsizip driver @@ -115,27 +116,19 @@ end function write_attributes(loc::Union{AG.AbstractGroup,AG.AbstractMDArray}) for name in attribute_names - @test AG.writeattribute(loc, name, name) + AG.writeattribute(loc, name, name) end for T in attribute_types - @test AG.writeattribute(loc, "$T", get_attribute_value(T)) + AG.writeattribute(loc, "$T", get_attribute_value(T)) end return nothing end function test_attributes(loc::Union{AG.AbstractGroup,AG.AbstractMDArray}) for name in attribute_names @test isequal(AG.readattribute(loc, name), name) - if !isequal(AG.readattribute(loc, name), name) - @show loc name - @assert false - end end for T in attribute_types @test isequal(AG.readattribute(loc, "$T"), get_attribute_value(T)) - if !isequal(AG.readattribute(loc, "$T"), get_attribute_value(T)) - @show loc T - @assert false - end end return nothing end @@ -216,15 +209,11 @@ end # @test AG.iswritable(mdarray) data = Float32[100 * x + y for y in 1:ny, x in 1:nx] - - success = AG.write(mdarray, data) - @test success + AG.write(mdarray, data) write_attributes(mdarray) - success = - AG.writemdarray(group, "primes", UInt8[2, 3, 5, 7, 251]) - @test success + AG.writemdarray(group, "primes", UInt8[2, 3, 5, 7, 251]) if drivername != "MEM" err = AG.close(dataset) @@ -280,42 +269,43 @@ end @test AG.getfullname(dimx) == "/group/x" @test AG.gettype(dimx) == "" @test AG.getdirection(dimx) == "" - xvar = AG.getindexingvariable(dimx) - @test AG.isnull(xvar) + @test_throws ErrorException AG.getindexingvariable(dimx) # TODO: setindexingvariable! # TODO: rename! mdarray1 = AG.openmdarrayfromfullname(root, "/group/mdarray") @test !AG.isnull(mdarray1) @test AG.getfullname(mdarray1) == "/group/mdarray" - @test AG.isnull( - AG.openmdarrayfromfullname(root, "/group/doesnotexist"), + @test_throws ErrorException AG.openmdarrayfromfullname( + root, + "/group/doesnotexist", ) - mdarray2 = AG.resolvemdarray(group, "mdarray", "") @test !AG.isnull(mdarray2) @test AG.getfullname(mdarray2) == "/group/mdarray" - @test AG.isnull(AG.resolvemdarray(group, "doesnotexist", "")) + @test_throws ErrorException AG.resolvemdarray( + group, + "doesnotexist", + "", + ) group1 = AG.opengroupfromfullname(root, "/group") @test !AG.isnull(group1) @test AG.getfullname(group1) == "/group" - @test AG.isnull( - AG.opengroupfromfullname(group, "/doesnotexist"), + @test_throws ErrorException AG.opengroupfromfullname( + group, + "/doesnotexist", ) - datatype = AG.getdatatype(mdarray) @test !AG.isnull(datatype) @test AG.getclass(datatype) == GDAL.GEDTC_NUMERIC @test AG.getnumericdatatype(datatype) == AG.GDT_Float32 data = Array{Float32}(undef, ny, nx) - success = AG.read!(mdarray, data) - @test success + AG.read!(mdarray, data) @test data == Float32[100 * x + y for y in 1:ny, x in 1:nx] data = AG.read(mdarray) - @test data !== nothing @test data == Float32[100 * x + y for y in 1:ny, x in 1:nx] test_attributes(mdarray) @@ -346,78 +336,89 @@ end ) do dataset @test !AG.isnull(dataset) - root = AG.getrootgroup(dataset) - @test !AG.isnull(root) - - rootname = AG.getname(root) - @test rootname == "/" - rootfullname = AG.getfullname(root) - @test rootfullname == "/" - - AG.creategroup(root, "group") do group - @test !AG.isnull(group) - @test AG.getname(group) == "group" - @test AG.getfullname(group) == "/group" - - @test AG.getgroupnames(root) == ["group"] - @test AG.getgroupnames(group) == [] - - write_attributes(group) - - nx, ny = 3, 4 - AG.createdimension(group, "x", "", "", nx) do dimx - @test !AG.isnull(dimx) - AG.createdimension(group, "y", "", "", ny) do dimy - @test !AG.isnull(dimy) - - AG.getdimensions(root) do dims - @test dims == [] - end - AG.getdimensions(group) do dimensions - @test length(dimensions) == 2 - end - - AG.extendeddatatypecreate(Float32) do datatype - @test !AG.isnull(datatype) - - AG.createmdarray( - group, - "mdarray", - [dimx, dimy], - datatype, - mdarraycreateoptions, - ) do mdarray - @test !AG.isnull(mdarray) - - @test AG.getmdarraynames(root) == [] - @test AG.getmdarraynames(group) == - ["mdarray"] - - @test AG.getvectorlayernames(root) == [] - @test AG.getvectorlayernames(group) == - [] - - @test AG.getstructuralinfo(group) == [] - - # @test AG.iswritable(mdarray) - - data = Float32[ - 100 * x + y for y in 1:ny, x in 1:nx - ] - - success = AG.write(mdarray, data) - @test success + AG.getrootgroup(dataset) do root + @test !AG.isnull(root) + + rootname = AG.getname(root) + @test rootname == "/" + rootfullname = AG.getfullname(root) + @test rootfullname == "/" + + AG.creategroup(root, "group") do group + @test !AG.isnull(group) + @test AG.getname(group) == "group" + @test AG.getfullname(group) == "/group" + + @test AG.getgroupnames(root) == ["group"] + @test AG.getgroupnames(group) == [] + + write_attributes(group) + + nx, ny = 3, 4 + AG.createdimension(group, "x", "", "", nx) do dimx + @test !AG.isnull(dimx) + AG.createdimension( + group, + "y", + "", + "", + ny, + ) do dimy + @test !AG.isnull(dimy) + + AG.getdimensions(root) do dims + @test dims == [] + end + AG.getdimensions(group) do dimensions + @test length(dimensions) == 2 + end - write_attributes(mdarray) + AG.extendeddatatypecreate( + Float32, + ) do datatype + @test !AG.isnull(datatype) - success = AG.writemdarray( + AG.createmdarray( group, - "primes", - UInt8[2, 3, 5, 7, 251], - ) - @test success - - return + "mdarray", + [dimx, dimy], + datatype, + mdarraycreateoptions, + ) do mdarray + @test !AG.isnull(mdarray) + + @test AG.getmdarraynames(root) == [] + @test AG.getmdarraynames(group) == + ["mdarray"] + + @test AG.getvectorlayernames( + root, + ) == [] + @test AG.getvectorlayernames( + group, + ) == [] + + @test AG.getstructuralinfo(group) == + [] + + # @test AG.iswritable(mdarray) + + data = Float32[ + 100 * x + y for y in 1:ny, + x in 1:nx + ] + AG.write(mdarray, data) + + write_attributes(mdarray) + + AG.writemdarray( + group, + "primes", + UInt8[2, 3, 5, 7, 251], + ) + + return + end end end end @@ -473,8 +474,9 @@ end @test AG.getfullname(dimx) == "/group/x" @test AG.gettype(dimx) == "" @test AG.getdirection(dimx) == "" - AG.getindexingvariable(dimx) do xvar - @test AG.isnull(xvar) + @test_throws ErrorException AG.getindexingvariable( + dimx, + ) do xvar end # TODO: setindexingvariable! # TODO: rename! @@ -494,11 +496,10 @@ end @test AG.getfullname(mdarray1) == "/group/mdarray" end - AG.openmdarrayfromfullname( + @test_throws ErrorException AG.openmdarrayfromfullname( root, "/group/doesnotexist", ) do doesnotexist - @test AG.isnull(doesnotexist) end AG.resolvemdarray( @@ -510,12 +511,11 @@ end @test AG.getfullname(mdarray2) == "/group/mdarray" end - AG.resolvemdarray( + @test_throws ErrorException AG.resolvemdarray( root, "doesnotexist", "", ) do doesnotexist - @test AG.isnull(doesnotexist) end AG.opengroupfromfullname( @@ -526,22 +526,19 @@ end @test AG.getfullname(group1) == "/group" end - AG.opengroupfromfullname( + @test_throws ErrorException AG.opengroupfromfullname( root, "/doesnotexist", ) do doesnotexist - @test AG.isnull(doesnotexist) end data = Array{Float32}(undef, ny, nx) - success = AG.read!(mdarray, data) - @test success + AG.read!(mdarray, data) @test data == Float32[ 100 * x + y for y in 1:ny, x in 1:nx ] data = AG.read(mdarray) - @test data !== nothing @test data == Float32[ 100 * x + y for y in 1:ny, x in 1:nx ] From 77d22bf59b370e69f23569479160d1e900b577f5 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 20 Aug 2024 13:54:19 -0400 Subject: [PATCH 16/19] Change constructor API: Take parent as WeakRef --- src/mdarray/attribute.jl | 6 ++-- src/mdarray/dimension.jl | 4 +-- src/mdarray/global.jl | 4 +-- src/mdarray/group.jl | 60 +++++++++++++++++------------------- src/mdarray/mdarray.jl | 66 ++++++++++++++++------------------------ src/mdarray/types.jl | 38 ++++++++++------------- src/types.jl | 6 +++- test/test_mdarray.jl | 7 +++++ 8 files changed, 89 insertions(+), 102 deletions(-) diff --git a/src/mdarray/attribute.jl b/src/mdarray/attribute.jl index 4cdd8fde..fc6c6d84 100644 --- a/src/mdarray/attribute.jl +++ b/src/mdarray/attribute.jl @@ -177,9 +177,8 @@ function getdimensions( dimensionscountref = Ref{Csize_t}() dimensionshptr = GDAL.gdalattributegetdimensions(attribute, dimensionscountref) - dataset = attribute.dataset.value dimensions = AbstractDimension[ - IDimension(unsafe_load(dimensionshptr, n), dataset) for + IDimension(unsafe_load(dimensionshptr, n), attribute.dataset) for n in 1:dimensionscountref[] ] GDAL.vsifree(dimensionshptr) @@ -193,9 +192,8 @@ function unsafe_getdimensions( dimensionscountref = Ref{Csize_t}() dimensionshptr = GDAL.gdalattributegetdimensions(attribute, dimensionscountref) - dataset = attribute.dataset.value dimensions = AbstractDimension[ - Dimension(unsafe_load(dimensionshptr, n), dataset) for + Dimension(unsafe_load(dimensionshptr, n), attribute.dataset) for n in 1:dimensionscountref[] ] GDAL.vsifree(dimensionshptr) diff --git a/src/mdarray/dimension.jl b/src/mdarray/dimension.jl index 5e7ef0d7..5963c8a4 100644 --- a/src/mdarray/dimension.jl +++ b/src/mdarray/dimension.jl @@ -31,14 +31,14 @@ function unsafe_getindexingvariable( @assert !isnull(dimension) ptr = GDAL.gdaldimensiongetindexingvariable(dimension) ptr == C_NULL && error("Could not get indexing variable for dimension") - return MDArray(ptr, dimension.dataset.value) + return MDArray(ptr, dimension.dataset) end function getindexingvariable(dimension::AbstractDimension)::AbstractMDArray @assert !isnull(dimension) ptr = GDAL.gdaldimensiongetindexingvariable(dimension) ptr == C_NULL && error("Could not get indexing variable for dimension") - return IMDArray(ptr, dimension.dataset.value) + return IMDArray(ptr, dimension.dataset) end function setindexingvariable!( diff --git a/src/mdarray/global.jl b/src/mdarray/global.jl index 248e433c..e950bb7f 100644 --- a/src/mdarray/global.jl +++ b/src/mdarray/global.jl @@ -100,10 +100,10 @@ end function unsafe_getrootgroup(dataset::AbstractDataset)::AbstractGroup @assert !isnull(dataset) - return Group(GDAL.gdaldatasetgetrootgroup(dataset), dataset) + return Group(GDAL.gdaldatasetgetrootgroup(dataset), WeakRef(dataset)) end function getrootgroup(dataset::AbstractDataset)::AbstractGroup @assert !isnull(dataset) - return Group(GDAL.gdaldatasetgetrootgroup(dataset), dataset) + return Group(GDAL.gdaldatasetgetrootgroup(dataset), WeakRef(dataset)) end diff --git a/src/mdarray/group.jl b/src/mdarray/group.jl index 0011a567..45d3c6b3 100644 --- a/src/mdarray/group.jl +++ b/src/mdarray/group.jl @@ -26,7 +26,7 @@ function unsafe_openmdarray( @assert !isnull(group) ptr = GDAL.gdalgroupopenmdarray(group, name, CSLConstListWrapper(options)) ptr == C_NULL && error("Could not open mdarray \"$name\"") - return MDArray(ptr, group.dataset.value) + return MDArray(ptr, group.dataset) end function openmdarray( @@ -37,7 +37,7 @@ function openmdarray( @assert !isnull(group) ptr = GDAL.gdalgroupopenmdarray(group, name, CSLConstListWrapper(options)) ptr == C_NULL && error("Could not open mdarray \"$name\"") - return IMDArray(ptr, group.dataset.value) + return IMDArray(ptr, group.dataset) end function getgroupnames( @@ -56,7 +56,7 @@ function unsafe_opengroup( @assert !isnull(group) ptr = GDAL.gdalgroupopengroup(group, name, CSLConstListWrapper(options)) ptr == C_NULL && error("Could no open group \"$name\"") - return Group(ptr, group.dataset.value) + return Group(ptr, group.dataset) end function opengroup( @@ -67,7 +67,7 @@ function opengroup( @assert !isnull(group) ptr = GDAL.gdalgroupopengroup(group, name, CSLConstListWrapper(options)) ptr == C_NULL && error("Could no open group \"$name\"") - return IGroup(ptr, group.dataset.value) + return IGroup(ptr, group.dataset) end function getvectorlayernames( @@ -120,9 +120,8 @@ function unsafe_getdimensions( dimensionscountref, CSLConstListWrapper(options), ) - dataset = group.dataset.value dimensions = AbstractDimension[ - Dimension(unsafe_load(dimensionshptr, n), dataset) for + Dimension(unsafe_load(dimensionshptr, n), group.dataset) for n in 1:dimensionscountref[] ] GDAL.vsifree(dimensionshptr) @@ -140,9 +139,8 @@ function getdimensions( dimensionscountref, CSLConstListWrapper(options), ) - dataset = group.dataset.value dimensions = AbstractDimension[ - IDimension(unsafe_load(dimensionshptr, n), dataset) for + IDimension(unsafe_load(dimensionshptr, n), group.dataset) for n in 1:dimensionscountref[] ] GDAL.vsifree(dimensionshptr) @@ -157,7 +155,7 @@ function unsafe_creategroup( @assert !isnull(group) ptr = GDAL.gdalgroupcreategroup(group, name, CSLConstListWrapper(options)) ptr == C_NULL && error("Could not create group \"$name\"") - return Group(ptr, group.dataset.value) + return Group(ptr, group.dataset) end function creategroup( @@ -168,7 +166,7 @@ function creategroup( @assert !isnull(group) ptr = GDAL.gdalgroupcreategroup(group, name, CSLConstListWrapper(options)) ptr == C_NULL && error("Could not create group \"$name\"") - return IGroup(ptr, group.dataset.value) + return IGroup(ptr, group.dataset) end function deletegroup( @@ -199,7 +197,7 @@ function unsafe_createdimension( CSLConstListWrapper(options), ) ptr == C_NULL && error("Could not create dimension \"$name\"") - return Dimension(ptr, group.dataset.value) + return Dimension(ptr, group.dataset) end function createdimension( @@ -220,7 +218,7 @@ function createdimension( CSLConstListWrapper(options), ) ptr == C_NULL && error("Could not create dimension \"$name\"") - return IDimension(ptr, group.dataset.value) + return IDimension(ptr, group.dataset) end function unsafe_createmdarray( @@ -242,7 +240,7 @@ function unsafe_createmdarray( CSLConstListWrapper(options), ) ptr == C_NULL && error("Could not create mdarray \"$name\"") - return MDArray(ptr, group.dataset.value) + return MDArray(ptr, group.dataset) end function createmdarray( @@ -264,7 +262,7 @@ function createmdarray( CSLConstListWrapper(options), ) ptr == C_NULL && error("Could not create mdarray \"$name\"") - return IMDArray(ptr, group.dataset.value) + return IMDArray(ptr, group.dataset) end function deletemdarray( @@ -302,7 +300,7 @@ function unsafe_openmdarrayfromfullname( CSLConstListWrapper(options), ) ptr == C_NULL && error("Could not open mdarray \"$fullname\"") - return MDArray(ptr, group.dataset.value) + return MDArray(ptr, group.dataset) end function openmdarrayfromfullname( @@ -317,7 +315,7 @@ function openmdarrayfromfullname( CSLConstListWrapper(options), ) ptr == C_NULL && error("Could not open mdarray \"$fullname\"") - return IMDArray(ptr, group.dataset.value) + return IMDArray(ptr, group.dataset) end function unsafe_resolvemdarray( @@ -334,7 +332,7 @@ function unsafe_resolvemdarray( CSLConstListWrapper(options), ) ptr == C_NULL && error("Could not resolve mdarray \"$name\"") - return MDArray(ptr, group.dataset.value) + return MDArray(ptr, group.dataset) end function resolvemdarray( @@ -351,7 +349,7 @@ function resolvemdarray( CSLConstListWrapper(options), ) ptr == C_NULL && error("Could not resolve mdarray \"$name\"") - return IMDArray(ptr, group.dataset.value) + return IMDArray(ptr, group.dataset) end function unsafe_opengroupfromfullname( @@ -366,7 +364,7 @@ function unsafe_opengroupfromfullname( CSLConstListWrapper(options), ) ptr == C_NULL && error("Could not open group \"$fullname\"") - return Group(ptr, group.dataset.value) + return Group(ptr, group.dataset) end function opengroupfromfullname( @@ -381,7 +379,7 @@ function opengroupfromfullname( CSLConstListWrapper(options), ) ptr == C_NULL && error("Could not open group \"$fullname\"") - return IGroup(ptr, group.dataset.value) + return IGroup(ptr, group.dataset) end # function unsafe_opendimensionfromfullname( @@ -391,7 +389,7 @@ end # )::AbstractDimension # @assert !isnull(group) # return Dimension( -# GDAL.gdalgroupopendimensionfromfullname(group, fullname, CSLConstListWrapper(options)), group.dataset.value +# GDAL.gdalgroupopendimensionfromfullname(group, fullname, CSLConstListWrapper(options)), group.dataset # ) # end # @@ -402,7 +400,7 @@ end # )::AbstractDimension # @assert !isnull(group) # return IDimension( -# GDAL.gdalgroupopendimensionfromfullname(group, fullname, CSLConstListWrapper(options)), group.dataset.value +# GDAL.gdalgroupopendimensionfromfullname(group, fullname, CSLConstListWrapper(options)), group.dataset # ) # end @@ -425,7 +423,7 @@ function unsafe_subsetdimensionfromselection( selection, CSLConstListWrapper(options), ), - group.dataset.value, + group.dataset, ) end @@ -441,7 +439,7 @@ function subsetdimensionfromselection( selection, CSLConstListWrapper(options), ), - group.dataset.value, + group.dataset, ) end @@ -454,7 +452,7 @@ function unsafe_getattribute( @assert !isnull(group) ptr = GDAL.gdalgroupgetattribute(group, name) ptr == C_NULL && error("Could not open attribute \"$name\"") - return Attribute(ptr, group.dataset.value) + return Attribute(ptr, group.dataset) end function getattribute( @@ -464,7 +462,7 @@ function getattribute( @assert !isnull(group) ptr = GDAL.gdalgroupgetattribute(group, name) ptr == C_NULL && error("Could not open attribute \"$name\"") - return IAttribute(ptr, group.dataset.value) + return IAttribute(ptr, group.dataset) end function unsafe_getattributes( @@ -475,9 +473,8 @@ function unsafe_getattributes( count = Ref{Csize_t}() ptr = GDAL.gdalgroupgetattributes(group, count, CSLConstListWrapper(options)) - dataset = group.dataset.value attributes = AbstractAttribute[ - Attribute(unsafe_load(ptr, n), dataset) for n in 1:count[] + Attribute(unsafe_load(ptr, n), group.dataset) for n in 1:count[] ] GDAL.vsifree(ptr) return attributes @@ -491,9 +488,8 @@ function getattributes( count = Ref{Csize_t}() ptr = GDAL.gdalgroupgetattributes(group, count, CSLConstListWrapper(options)) - dataset = group.dataset.value attributes = AbstractAttribute[ - IAttribute(unsafe_load(ptr, n), dataset) for n in 1:count[] + IAttribute(unsafe_load(ptr, n), group.dataset) for n in 1:count[] ] GDAL.vsifree(ptr) return attributes @@ -517,7 +513,7 @@ function unsafe_createattribute( CSLConstListWrapper(options), ) ptr == C_NULL && error("Could not create attribute \"$name\"") - return Attribute(ptr, group.dataset.value) + return Attribute(ptr, group.dataset) end function createattribute( @@ -538,7 +534,7 @@ function createattribute( CSLConstListWrapper(options), ) ptr == C_NULL && error("Could not create attribute \"$name\"") - return IAttribute(ptr, group.dataset.value) + return IAttribute(ptr, group.dataset) end function deleteattribute( diff --git a/src/mdarray/mdarray.jl b/src/mdarray/mdarray.jl index 999ee208..f5ea4556 100644 --- a/src/mdarray/mdarray.jl +++ b/src/mdarray/mdarray.jl @@ -1,6 +1,6 @@ # GDALMDArray -function MDArray(ptr::GDAL.GDALMDArrayH, dataset::AbstractDataset) +function MDArray(ptr::GDAL.GDALMDArrayH, dataset::WeakRef) @assert ptr != C_NULL datatype = IExtendedDataType(GDAL.gdalmdarraygetdatatype(ptr)) class = getclass(datatype) @@ -10,7 +10,7 @@ function MDArray(ptr::GDAL.GDALMDArrayH, dataset::AbstractDataset) return MDArray{T,D}(ptr, dataset) end -function IMDArray(ptr::GDAL.GDALMDArrayH, dataset::AbstractDataset) +function IMDArray(ptr::GDAL.GDALMDArrayH, dataset::WeakRef) @assert ptr != C_NULL datatype = IExtendedDataType(GDAL.gdalmdarraygetdatatype(ptr)) class = getclass(datatype) @@ -218,7 +218,7 @@ function unsafe_getview( @assert !isnull(mdarray) ptr = GDAL.gdalmdarraygetview(mdarray, viewexpr) ptr == C_NULL && error("Could not get view \"$vierexpr\"") - return MDArray(ptr, mdarray.dataset.value) + return MDArray(ptr, mdarray.dataset) end function getview( @@ -228,7 +228,7 @@ function getview( @assert !isnull(mdarray) ptr = GDAL.gdalmdarraygetview(mdarray, viewexpr) ptr == C_NULL && error("Could not get view \"$vierexpr\"") - return IMDArray(ptr, mdarray.dataset.value) + return IMDArray(ptr, mdarray.dataset) end function unsafe_getindex( @@ -272,14 +272,14 @@ function unsafe_transpose(mdarray::AbstractMDArray)::AbstractMDArray @assert !isnull(mdarray) ptr = GDAL.gdalmdarraytranspose(mdarray) ptr == C_NULL && error("Could not transpose mdarray") - return MDArray(ptr, mdarray.dataset.value) + return MDArray(ptr, mdarray.dataset) end function transpose(mdarray::AbstractMDArray)::AbstractMDArray @assert !isnull(mdarray) ptr = GDAL.gdalmdarraytranspose(mdarray) ptr == C_NULL && error("Could not transpose mdarray") - return IMDArray(ptr, mdarray.dataset.value) + return IMDArray(ptr, mdarray.dataset) end function unsafe_getunscaled( @@ -296,7 +296,7 @@ function unsafe_getunscaled( overriddendstnodata, ) ptr == C_NULL && error("Could not get unscaled mdarray") - return MDArray(ptr, mdarray.dataset.value) + return MDArray(ptr, mdarray.dataset) end function getunscaled( @@ -313,7 +313,7 @@ function getunscaled( overriddendstnodata, ) ptr == C_NULL && error("Could not get unscaled mdarray") - return IMDArray(ptr, mdarray.dataset.value) + return IMDArray(ptr, mdarray.dataset) end function unsafe_getmask( @@ -323,7 +323,7 @@ function unsafe_getmask( @assert !isnull(mdarray) ptr = GDAL.gdalmdarraygetmask(mdarray, CSLConstListWrapper(options)) ptr == C_NULL && error("Could not get mask for mdarray") - return MDArray(ptr, mdarray.dataset.value) + return MDArray(ptr, mdarray.dataset) end function getmask( @@ -333,7 +333,7 @@ function getmask( @assert !isnull(mdarray) ptr = GDAL.gdalmdarraygetmask(mdarray, CSLConstListWrapper(options)) ptr == C_NULL && error("Could not get mask for mdarray") - return IMDArray(ptr, mdarray.dataset.value) + return IMDArray(ptr, mdarray.dataset) end # TODO: Wrap GDAL.GDALRIOResampleAlg @@ -354,7 +354,7 @@ function unsafe_getresampled( CSLConstListWrapper(options), ) ptr == C_NULL && error("Could not get resampled mdarray") - return MDArray(ptr, mdarray.dataset.value) + return MDArray(ptr, mdarray.dataset) end function getresampled( @@ -374,7 +374,7 @@ function getresampled( CSLConstListWrapper(options), ) ptr == C_NULL && error("Could not get resampled mdarray") - return IMDArray(ptr, mdarray.dataset.value) + return IMDArray(ptr, mdarray.dataset) end function unsafe_getgridded( @@ -395,7 +395,7 @@ function unsafe_getgridded( CSLConstListWrapper(options), ) ptr == C_NULL && error("Could not get gridded mdarray") - return MDArray(ptr, mdarray.dataset.value) + return MDArray(ptr, mdarray.dataset) end function getgridded( @@ -416,7 +416,7 @@ function getgridded( CSLConstListWrapper(options), ) ptr == C_NULL && error("Could not get gridded mdarray") - return IMDArray(ptr, mdarray.dataset.value) + return IMDArray(ptr, mdarray.dataset) end function unsafe_asclassicdataset( @@ -546,10 +546,9 @@ function getcoordinatevariables( count = Ref{Csize_t}() coordinatevariablesptr = GDAL.gdalmdarraygetcoordinatevariables(mdarray, count) - dataset = mdarray.dataset.value coordinatevariables = AbstractMDArray[ - IMDArray(unsafe_load(coordinatevariablesptr, n), dataset) for - n in 1:count[] + IMDArray(unsafe_load(coordinatevariablesptr, n), mdarray.dataset) + for n in 1:count[] ] GDAL.vsifree(coordinatevariablesptr) return coordinatevariables @@ -607,7 +606,7 @@ end function getrootgroup(mdarray::AbstractMDArray)::AbstractGroup @assert !isnull(mdarray) - return Group(GDAL.gdalmdarraygetrootgroup(mdarray), mdarray.dataset.value) + return Group(GDAL.gdalmdarraygetrootgroup(mdarray), mdarray.dataset) end ################################################################################ @@ -646,9 +645,8 @@ function getdimensions( @assert !isnull(mdarray) dimensionscountref = Ref{Csize_t}() dimensionshptr = GDAL.gdalmdarraygetdimensions(mdarray, dimensionscountref) - dataset = mdarray.dataset.value dimensions = AbstractDimension[ - IDimension(unsafe_load(dimensionshptr, n), dataset) for + IDimension(unsafe_load(dimensionshptr, n), mdarray.dataset) for n in 1:dimensionscountref[] ] GDAL.vsifree(dimensionshptr) @@ -661,9 +659,8 @@ function unsafe_getdimensions( @assert !isnull(mdarray) dimensionscountref = Ref{Csize_t}() dimensionshptr = GDAL.gdalmdarraygetdimensions(mdarray, dimensionscountref) - dataset = mdarray.dataset.value dimensions = AbstractDimension[ - Dimension(unsafe_load(dimensionshptr, n), dataset) for + Dimension(unsafe_load(dimensionshptr, n), mdarray.dataset) for n in 1:dimensionscountref[] ] GDAL.vsifree(dimensionshptr) @@ -689,17 +686,10 @@ end Base.eltype(mdarray::AbstractMDArray{T}) where {T} = T -function getblocksize( - mdarray::AbstractMDArray, - options::OptionList = nothing, -)::AbstractVector{Int64} +function getblocksize(mdarray::AbstractMDArray)::AbstractVector{Int64} @assert !isnull(mdarray) count = Ref{Csize_t}() - blocksizeptr = GDAL.gdalmdarraygetblocksize( - mdarray, - count, - CSLConstListWrapper(options), - ) + blocksizeptr = GDAL.gdalmdarraygetblocksize(mdarray, count) blocksize = Int64[unsafe_load(blocksizeptr, n) for n in count[]:-1:1] GDAL.vsifree(blocksizeptr) return blocksize @@ -928,7 +918,7 @@ function unsafe_getattribute( @assert !isnull(mdarray) ptr = GDAL.gdalmdarraygetattribute(mdarray, name) ptr == C_NULL && error("Could not get attribute \"$name\"") - return Attribute(ptr, mdarray.dataset.value) + return Attribute(ptr, mdarray.dataset) end function getattribute( @@ -938,7 +928,7 @@ function getattribute( @assert !isnull(mdarray) ptr = GDAL.gdalmdarraygetattribute(mdarray, name) ptr == C_NULL && error("Could not get attribute \"$name\"") - return IAttribute(ptr, mdarray.dataset.value) + return IAttribute(ptr, mdarray.dataset) end function unsafe_getattributes( @@ -952,9 +942,8 @@ function unsafe_getattributes( count, CSLConstListWrapper(options), ) - dataset = mdarray.dataset.value attributes = AbstractAttribute[ - Attribute(unsafe_load(ptr, n), dataset) for n in 1:count[] + Attribute(unsafe_load(ptr, n), mdarray.dataset) for n in 1:count[] ] GDAL.vsifree(ptr) return attributes @@ -971,9 +960,8 @@ function getattributes( count, CSLConstListWrapper(options), ) - dataset = mdarray.dataset.value attributes = AbstractAttribute[ - IAttribute(unsafe_load(ptr, n), dataset) for n in 1:count[] + IAttribute(unsafe_load(ptr, n), mdarray.dataset) for n in 1:count[] ] GDAL.vsifree(ptr) return attributes @@ -997,7 +985,7 @@ function unsafe_createattribute( CSLConstListWrapper(options), ) ptr == C_NULL && error("Could not create attribute \"$name\"") - return Attribute(ptr, mdarray.dataset.value) + return Attribute(ptr, mdarray.dataset) end function createattribute( @@ -1018,7 +1006,7 @@ function createattribute( CSLConstListWrapper(options), ) ptr == C_NULL && error("Could not create attribute \"$name\"") - return IAttribute(ptr, mdarray.dataset.value) + return IAttribute(ptr, mdarray.dataset) end function deleteattribute( diff --git a/src/mdarray/types.jl b/src/mdarray/types.jl index 715167e5..9b4a8276 100644 --- a/src/mdarray/types.jl +++ b/src/mdarray/types.jl @@ -56,8 +56,8 @@ mutable struct Group <: AbstractGroup ptr::GDAL.GDALGroupH dataset::WeakRef # AbstractDataset - function Group(ptr::GDAL.GDALGroupH, dataset::AbstractDataset) - group = new(ptr, WeakRef(dataset)) + function Group(ptr::GDAL.GDALGroupH, dataset::WeakRef) + group = new(ptr, dataset) add_child!(dataset, group) return group end @@ -67,8 +67,8 @@ mutable struct IGroup <: AbstractGroup ptr::GDAL.GDALGroupH dataset::WeakRef # AbstractDataset - function IGroup(ptr::GDAL.GDALGroupH, dataset::AbstractDataset) - group = new(ptr, WeakRef(dataset)) + function IGroup(ptr::GDAL.GDALGroupH, dataset::WeakRef) + group = new(ptr, dataset) add_child!(dataset, group) ptr != C_NULL && finalizer(destroy, group) return group @@ -79,13 +79,10 @@ mutable struct MDArray{T,D} <: AbstractMDArray{T,D} ptr::GDAL.GDALMDArrayH dataset::WeakRef # AbstractDataset - function MDArray{T,D}( - ptr::GDAL.GDALMDArrayH, - dataset::AbstractDataset, - ) where {T,D} + function MDArray{T,D}(ptr::GDAL.GDALMDArrayH, dataset::WeakRef) where {T,D} T::Type D::Int - mdarray = new{T,D}(ptr, WeakRef(dataset)) + mdarray = new{T,D}(ptr, dataset) add_child!(dataset, mdarray) return mdarray end @@ -95,13 +92,10 @@ mutable struct IMDArray{T,D} <: AbstractMDArray{T,D} ptr::GDAL.GDALMDArrayH dataset::WeakRef # AbstractDataset - function IMDArray{T,D}( - ptr::GDAL.GDALMDArrayH, - dataset::AbstractDataset, - ) where {T,D} + function IMDArray{T,D}(ptr::GDAL.GDALMDArrayH, dataset::WeakRef) where {T,D} T::Type D::Int - mdarray = new{T,D}(ptr, WeakRef(dataset)) + mdarray = new{T,D}(ptr, dataset) add_child!(dataset, mdarray) ptr != C_NULL && finalizer(destroy, mdarray) return mdarray @@ -112,8 +106,8 @@ mutable struct Attribute <: AbstractAttribute ptr::GDAL.GDALAttributeH dataset::WeakRef # AbstractDataset - function Attribute(ptr::GDAL.GDALAttributeH, dataset::AbstractDataset) - attribute = new(ptr, WeakRef(dataset)) + function Attribute(ptr::GDAL.GDALAttributeH, dataset::WeakRef) + attribute = new(ptr, dataset) add_child!(dataset, attribute) return attribute end @@ -123,8 +117,8 @@ mutable struct IAttribute <: AbstractAttribute ptr::GDAL.GDALAttributeH dataset::WeakRef # AbstractDataset - function IAttribute(ptr::GDAL.GDALAttributeH, dataset::AbstractDataset) - attribute = new(ptr, WeakRef(dataset)) + function IAttribute(ptr::GDAL.GDALAttributeH, dataset::WeakRef) + attribute = new(ptr, dataset) add_child!(dataset, attribute) ptr != C_NULL && finalizer(destroy, attribute) return attribute @@ -135,8 +129,8 @@ mutable struct Dimension <: AbstractDimension ptr::GDAL.GDALDimensionH dataset::WeakRef # AbstractDataset - function Dimension(ptr::GDAL.GDALDimensionH, dataset::AbstractDataset) - dimension = new(ptr, WeakRef(dataset)) + function Dimension(ptr::GDAL.GDALDimensionH, dataset::WeakRef) + dimension = new(ptr, dataset) add_child!(dataset, dimension) return dimension end @@ -146,8 +140,8 @@ mutable struct IDimension <: AbstractDimension ptr::GDAL.GDALDimensionH dataset::WeakRef # AbstractDataset - function IDimension(ptr::GDAL.GDALDimensionH, dataset::AbstractDataset) - dimension = new(ptr, WeakRef(dataset)) + function IDimension(ptr::GDAL.GDALDimensionH, dataset::WeakRef) + dimension = new(ptr, dataset) add_child!(dataset, dimension) ptr != C_NULL && finalizer(destroy, dimension) return dimension diff --git a/src/types.jl b/src/types.jl index 1af67143..1bc768cf 100644 --- a/src/types.jl +++ b/src/types.jl @@ -66,8 +66,12 @@ mutable struct IDataset <: AbstractDataset end end -function add_child!(dataset::AbstractDataset, obj::Any)::Nothing +function add_child!(dataset::WeakRef, obj::Any)::Nothing isnull(obj) && return nothing + dataset = dataset.value + # It is fine if the dataset does not exist any more + isnothing(dataset) && return nothing + dataset::AbstractDataset @assert !isnull(dataset) isnothing(dataset.children) && return nothing push!(dataset.children, WeakRef(obj)) diff --git a/test/test_mdarray.jl b/test/test_mdarray.jl index 85b29a17..12347d95 100644 --- a/test/test_mdarray.jl +++ b/test/test_mdarray.jl @@ -153,6 +153,7 @@ end drivercreateoptions, ) @test !AG.isnull(dataset) + @test match(r"GDAL Dataset", string(dataset)) !== nothing files = AG.filelist(dataset) if drivername in ["MEM"] @@ -165,6 +166,7 @@ end root = AG.getrootgroup(dataset) @test !AG.isnull(root) + @test match(r"ArchGDAL.Group", string(root)) !== nothing rootname = AG.getname(root) @test rootname == "/" rootfullname = AG.getfullname(root) @@ -172,6 +174,7 @@ end group = AG.creategroup(root, "group") @test !AG.isnull(group) + @test match(r"ArchGDAL.IGroup", string(group)) !== nothing @test AG.getname(group) == "group" @test AG.getfullname(group) == "/group" @@ -183,11 +186,14 @@ end nx, ny = 3, 4 dimx = AG.createdimension(group, "x", "", "", nx) @test !AG.isnull(dimx) + @test match(r"ArchGDAL.IDimension", string(dimx)) !== nothing dimy = AG.createdimension(group, "y", "", "", ny) @test !AG.isnull(dimy) datatype = AG.extendeddatatypecreate(Float32) @test !AG.isnull(datatype) + @test match(r"ArchGDAL.IExtendedDataType", string(datatype)) !== + nothing mdarray = AG.createmdarray( group, @@ -197,6 +203,7 @@ end mdarraycreateoptions, ) @test !AG.isnull(mdarray) + @test match(r"ArchGDAL.IMDArray", string(mdarray)) !== nothing @test AG.getmdarraynames(root) == [] @test AG.getmdarraynames(group) == ["mdarray"] From 36640fa12bb9ba8b177f936c73346d1ddf112c44 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 20 Aug 2024 17:27:36 -0400 Subject: [PATCH 17/19] Reverse array indices --- src/mdarray/group.jl | 4 ++-- src/mdarray/highlevel.jl | 3 +-- src/mdarray/mdarray.jl | 18 +++++++++--------- test/test_mdarray.jl | 18 +++++++++--------- 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/mdarray/group.jl b/src/mdarray/group.jl index 45d3c6b3..6d56e64e 100644 --- a/src/mdarray/group.jl +++ b/src/mdarray/group.jl @@ -235,7 +235,7 @@ function unsafe_createmdarray( group, name, length(dimensions), - DimensionHList(dimensions), + DimensionHList(reverse(dimensions)), datatype, CSLConstListWrapper(options), ) @@ -257,7 +257,7 @@ function createmdarray( group, name, length(dimensions), - DimensionHList(dimensions), + DimensionHList(reverse(dimensions)), datatype, CSLConstListWrapper(options), ) diff --git a/src/mdarray/highlevel.jl b/src/mdarray/highlevel.jl index eb7be715..55ae1869 100644 --- a/src/mdarray/highlevel.jl +++ b/src/mdarray/highlevel.jl @@ -7,8 +7,7 @@ function writemdarray( options::OptionList = nothing, )::Nothing where {T<:NumericAttributeType,D} dimensions = AbstractDimension[ - createdimension(group, "$name.$d", "", "", size(value, d)) for - d in D:-1:1 + createdimension(group, "$name.$d", "", "", size(value, d)) for d in 1:D ] extendeddatatypecreate(T) do datatype createmdarray(group, name, dimensions, datatype, options) do mdarray diff --git a/src/mdarray/mdarray.jl b/src/mdarray/mdarray.jl index f5ea4556..a5ccb9d1 100644 --- a/src/mdarray/mdarray.jl +++ b/src/mdarray/mdarray.jl @@ -254,7 +254,7 @@ function unsafe_getindex( indices::Integer..., )::AbstractMDArray @assert !isnull(mdarray) - viewexpr = "[" * join(indices, ",") * "]" + viewexpr = "[" * join(reverse(indices), ",") * "]" return unsafe_getview(mdarray, viewexpr) end @@ -263,7 +263,7 @@ function getindex( indices::Integer..., )::AbstractMDArray @assert !isnull(mdarray) - viewexpr = "[" * join(indices, ",") * "]" + viewexpr = "[" * join(reverse(indices), ",") * "]" return getview(mdarray, viewexpr) end @@ -348,7 +348,7 @@ function unsafe_getresampled( ptr = GDAL.gdalmdarraygetresampled( mdarray, isnothing(newdims) ? 0 : length(newdims), - isnothing(newdims) ? C_NULL : DimensionHList(newdims), + isnothing(newdims) ? C_NULL : DimensionHList(reverse(newdims)), resamplealg, isnothing(targetsrs) ? C_NULL : targetsrs, CSLConstListWrapper(options), @@ -368,7 +368,7 @@ function getresampled( ptr = GDAL.gdalmdarraygetresampled( mdarray, isnothing(newdims) ? 0 : length(newdims), - isnothing(newdims) ? C_NULL : DimensionHList(newdims), + isnothing(newdims) ? C_NULL : DimensionHList(reverse(newdims)), resamplealg, isnothing(targetsrs) ? C_NULL : targetsrs, CSLConstListWrapper(options), @@ -563,7 +563,7 @@ function adviseread( @assert !isnull(mdarray) return GDAL.gdalmdarrayadviseread( mdarray, - isnothing(arraystartidx) ? C_NULL : arraystartidx, + isnothing(arraystartidx) ? C_NULL : reverse(arraystartidx), isnothing(count) ? C_NULL : count, CSLConstListWrapper(options), ) @@ -647,7 +647,7 @@ function getdimensions( dimensionshptr = GDAL.gdalmdarraygetdimensions(mdarray, dimensionscountref) dimensions = AbstractDimension[ IDimension(unsafe_load(dimensionshptr, n), mdarray.dataset) for - n in 1:dimensionscountref[] + n in dimensionscountref[]:-1:1 ] GDAL.vsifree(dimensionshptr) return dimensions @@ -661,7 +661,7 @@ function unsafe_getdimensions( dimensionshptr = GDAL.gdalmdarraygetdimensions(mdarray, dimensionscountref) dimensions = AbstractDimension[ Dimension(unsafe_load(dimensionshptr, n), mdarray.dataset) for - n in 1:dimensionscountref[] + n in dimensionscountref[]:-1:1 ] GDAL.vsifree(dimensionshptr) return dimensions @@ -670,7 +670,7 @@ end function Base.size(mdarray::AbstractMDArray) getdimensions(mdarray) do dimensions D = length(dimensions) - return ntuple(d -> getsize(dimensions[D-d+1]), D) + return ntuple(d -> getsize(dimensions[d]), D) end end @@ -800,7 +800,7 @@ end function read(mdarray::AbstractMDArray)::AbstractArray getdimensions(mdarray) do dimensions D = length(dimensions) - sz = [getsize(dimensions[d]) for d in D:-1:1] + sz = [getsize(dimensions[d]) for d in 1:D] getdatatype(mdarray) do datatype class = getclass(datatype) @assert class == GDAL.GEDTC_NUMERIC diff --git a/test/test_mdarray.jl b/test/test_mdarray.jl index 12347d95..102240d5 100644 --- a/test/test_mdarray.jl +++ b/test/test_mdarray.jl @@ -215,7 +215,7 @@ end # @test AG.iswritable(mdarray) - data = Float32[100 * x + y for y in 1:ny, x in 1:nx] + data = Float32[x + 100 * y for x in 1:nx, y in 1:ny] AG.write(mdarray, data) write_attributes(mdarray) @@ -308,12 +308,12 @@ end @test AG.getclass(datatype) == GDAL.GEDTC_NUMERIC @test AG.getnumericdatatype(datatype) == AG.GDT_Float32 - data = Array{Float32}(undef, ny, nx) + data = Array{Float32}(undef, nx, ny) AG.read!(mdarray, data) - @test data == Float32[100 * x + y for y in 1:ny, x in 1:nx] + @test data == Float32[x + 100 * y for x in 1:nx, y in 1:ny] data = AG.read(mdarray) - @test data == Float32[100 * x + y for y in 1:ny, x in 1:nx] + @test data == Float32[x + 100 * y for x in 1:nx, y in 1:ny] test_attributes(mdarray) @@ -411,8 +411,8 @@ end # @test AG.iswritable(mdarray) data = Float32[ - 100 * x + y for y in 1:ny, - x in 1:nx + x + 100 * y for x in 1:nx, + y in 1:ny ] AG.write(mdarray, data) @@ -539,15 +539,15 @@ end ) do doesnotexist end - data = Array{Float32}(undef, ny, nx) + data = Array{Float32}(undef, nx, ny) AG.read!(mdarray, data) @test data == Float32[ - 100 * x + y for y in 1:ny, x in 1:nx + x + 100 * y for x in 1:nx, y in 1:ny ] data = AG.read(mdarray) @test data == Float32[ - 100 * x + y for y in 1:ny, x in 1:nx + x + 100 * y for x in 1:nx, y in 1:ny ] test_attributes(mdarray) From 4744e949d4218fc5a85a27f457a90d8d801476c8 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 21 Aug 2024 11:16:30 -0400 Subject: [PATCH 18/19] Use tuples instead of vectors for array dimensions --- src/mdarray/attribute.jl | 22 +++++------ src/mdarray/group.jl | 13 +++--- src/mdarray/mdarray.jl | 85 ++++++++++++++++++++++------------------ src/mdarray/types.jl | 23 +++++++++-- test/test_mdarray.jl | 53 ++++++++++++++++++++----- 5 files changed, 127 insertions(+), 69 deletions(-) diff --git a/src/mdarray/attribute.jl b/src/mdarray/attribute.jl index fc6c6d84..25daac76 100644 --- a/src/mdarray/attribute.jl +++ b/src/mdarray/attribute.jl @@ -24,9 +24,9 @@ function getdimensionssize(attribute::AbstractAttribute)::NTuple{<:Any,Int} @assert !isnull(attribute) count = Ref{Csize_t}() sizeptr = GDAL.gdalattributegetdimensionssize(attribute, count) - size = ntuple(n -> unsafe_load(sizeptr, n), count[]) + size = reverse(ntuple(d -> Int(unsafe_load(sizeptr, d), count[]))) GDAL.vsifree(sizeptr) - return Int.(size) + return size end function readasraw(attribute::AbstractAttribute)::AbstractVector{UInt8} @@ -158,7 +158,7 @@ function gettotalelementscount(attribute::AbstractAttribute)::Int64 return Int64(GDAL.gdalattributegettotalelementscount(attribute)) end -function Base.length(attribute::AbstractAttribute) +function Base.length(attribute::AbstractAttribute)::Int @assert !isnull(attribute) return Int(gettotalelementscount(attribute)) end @@ -178,8 +178,8 @@ function getdimensions( dimensionshptr = GDAL.gdalattributegetdimensions(attribute, dimensionscountref) dimensions = AbstractDimension[ - IDimension(unsafe_load(dimensionshptr, n), attribute.dataset) for - n in 1:dimensionscountref[] + IDimension(unsafe_load(dimensionshptr, d), attribute.dataset) for + d in dimensionscountref[]:-1:1 ] GDAL.vsifree(dimensionshptr) return dimensions @@ -193,8 +193,8 @@ function unsafe_getdimensions( dimensionshptr = GDAL.gdalattributegetdimensions(attribute, dimensionscountref) dimensions = AbstractDimension[ - Dimension(unsafe_load(dimensionshptr, n), attribute.dataset) for - n in 1:dimensionscountref[] + Dimension(unsafe_load(dimensionshptr, d), attribute.dataset) for + d in dimensionscountref[]:-1:1 ] GDAL.vsifree(dimensionshptr) return dimensions @@ -215,7 +215,7 @@ end function getblocksize( attribute::AbstractAttribute, options::OptionList = nothing, -)::AbstractVector{Int64} +)::NTuple{<:Any,Int} @assert !isnull(attribute) count = Ref{Csize_t}() blocksizeptr = GDAL.gdalattributegetblocksize( @@ -223,7 +223,7 @@ function getblocksize( count, CSLConstListWrapper(options), ) - blocksize = Int64[unsafe_load(blocksizeptr, n) for n in 1:count[]] + blocksize = reverse(ntuple(d -> Int(unsafe_load(blocksizeptr, d)), count[])) GDAL.vsifree(blocksizeptr) return blocksize end @@ -231,7 +231,7 @@ end function getprocessingchunksize( attribute::AbstractAttribute, maxchunkmemory::Integer, -)::AbstractVector{Int64} +)::NTuple{<:AnyInt} @assert !isnull(attribute) count = Ref{Csize_t}() chunksizeptr = GDAL.gdalattributegetprocessingchunksize( @@ -239,7 +239,7 @@ function getprocessingchunksize( count, maxchunkmemory, ) - chunksize = Int64[unsafe_load(chunksizeptr, n) for n in 1:count[]] + chunksize = reverse(ntuple(d -> Int(unsafe_load(chunksizeptr, d)), count[])) GDAL.vsifree(chunksizeptr) return chunksize end diff --git a/src/mdarray/group.jl b/src/mdarray/group.jl index 6d56e64e..3db9ebd0 100644 --- a/src/mdarray/group.jl +++ b/src/mdarray/group.jl @@ -175,7 +175,6 @@ function deletegroup( options::OptionList = nothing, )::Bool @assert !isnull(group) - # TODO: Do we need to set group.ptr = C_NULL? return GDAL.gdalgroupdeletegroup(group, name, CSLConstListWrapper(options)) end @@ -224,7 +223,7 @@ end function unsafe_createmdarray( group::AbstractGroup, name::AbstractString, - dimensions::AbstractVector{<:AbstractDimension}, + dimensions::VectorLike{<:AbstractDimension}, datatype::AbstractExtendedDataType, options::OptionList = nothing, )::AbstractMDArray @@ -246,7 +245,7 @@ end function createmdarray( group::AbstractGroup, name::AbstractString, - dimensions::AbstractVector{<:AbstractDimension}, + dimensions::VectorLike{<:AbstractDimension}, datatype::AbstractExtendedDataType, options::OptionList = nothing, )::AbstractMDArray @@ -498,7 +497,7 @@ end function unsafe_createattribute( group::AbstractGroup, name::AbstractString, - dimensions::AbstractVector{<:Integer}, + dimensions::VectorLike{<:Integer}, datatype::AbstractExtendedDataType, options::OptionList = nothing, )::AbstractAttribute @@ -508,7 +507,7 @@ function unsafe_createattribute( group, name, length(dimensions), - dimensions, + reverse(dimensions), datatype, CSLConstListWrapper(options), ) @@ -519,7 +518,7 @@ end function createattribute( group::AbstractGroup, name::AbstractString, - dimensions::AbstractVector{<:Integer}, + dimensions::VectorLike{<:Integer}, datatype::AbstractExtendedDataType, options::OptionList = nothing, )::AbstractAttribute @@ -529,7 +528,7 @@ function createattribute( group, name, length(dimensions), - dimensions, + reverse(dimensions), datatype, CSLConstListWrapper(options), ) diff --git a/src/mdarray/mdarray.jl b/src/mdarray/mdarray.jl index a5ccb9d1..1231607d 100644 --- a/src/mdarray/mdarray.jl +++ b/src/mdarray/mdarray.jl @@ -134,7 +134,7 @@ end function resize!( mdarray::AbstractMDArray, - newdimsizes::AbstractVector{<:Integer}, + newdimsizes::VectorLike{<:Integer}, options::OptionList = nothing, )::Bool @assert !isnull(mdarray) @@ -339,7 +339,7 @@ end # TODO: Wrap GDAL.GDALRIOResampleAlg function unsafe_getresampled( mdarray::AbstractMDArray, - newdims::Union{Nothing,AbstractVector{<:AbstractDimension}}, + newdims::Union{Nothing,VectorLike{<:AbstractDimension}}, resamplealg::GDAL.GDALRIOResampleAlg, targetsrs::Union{Nothing,AbstractSpatialRef}, options::OptionList = nothing, @@ -359,7 +359,7 @@ end function getresampled( mdarray::AbstractMDArray, - newdims::Union{Nothing,AbstractVector{<:AbstractDimension}}, + newdims::Union{Nothing,VectorLike{<:AbstractDimension}}, resamplealg::GDAL.GDALRIOResampleAlg, targetsrs::Union{Nothing,AbstractSpatialRef}, options::OptionList = nothing, @@ -540,27 +540,34 @@ function clearstatistics(mdarray::AbstractMDArray)::Nothing end function getcoordinatevariables( - mdarray::AbstractMDArray, -)::AbstractVector{<:AbstractMDArray} + mdarray::AbstractMDArray{<:Any,D}, +)::NTuple{D,T where T<:AbstractMDArray} where {D} @assert !isnull(mdarray) count = Ref{Csize_t}() coordinatevariablesptr = GDAL.gdalmdarraygetcoordinatevariables(mdarray, count) - coordinatevariables = AbstractMDArray[ - IMDArray(unsafe_load(coordinatevariablesptr, n), mdarray.dataset) - for n in 1:count[] - ] + coordinatevariables = reverse( + ntuple( + d -> IMDArray( + unsafe_load(coordinatevariablesptr, d), + mdarray.dataset, + ), + count[], + ), + ) GDAL.vsifree(coordinatevariablesptr) return coordinatevariables end function adviseread( - mdarray::AbstractMDArray, - arraystartidx::Union{Nothing,AbstractVector{<:Integer}}, - count::Union{Nothing,AbstractVector{<:Integer}}, + mdarray::AbstractMDArray{<:Any,D}, + arraystartidx::Union{Nothing,IndexLike{D}}, + count::Union{Nothing,IndexLike{D}}, options::OptionList = nothing, -)::Bool +)::Bool where {D} @assert !isnull(mdarray) + @assert isnothing(arraystartix) ? true : length(arraystartidx) == D + @assert isnothing(count) ? true : length(count == D) return GDAL.gdalmdarrayadviseread( mdarray, isnothing(arraystartidx) ? C_NULL : reverse(arraystartidx), @@ -640,36 +647,40 @@ getdimensioncount(mdarray::AbstractMDArray{<:Any,D}) where {D} = D Base.ndims(mdarray::AbstractMDArray)::Int = getdimensioncount(mdarray) function getdimensions( - mdarray::AbstractMDArray, -)::AbstractVector{<:AbstractDimension} + mdarray::AbstractMDArray{<:Any,D}, +)::NTuple{D,T where T<:AbstractDimension} where {D} @assert !isnull(mdarray) dimensionscountref = Ref{Csize_t}() dimensionshptr = GDAL.gdalmdarraygetdimensions(mdarray, dimensionscountref) - dimensions = AbstractDimension[ - IDimension(unsafe_load(dimensionshptr, n), mdarray.dataset) for - n in dimensionscountref[]:-1:1 - ] + dimensions = reverse( + ntuple( + d -> + IDimension(unsafe_load(dimensionshptr, d), mdarray.dataset), + dimensionscountref[], + ), + ) GDAL.vsifree(dimensionshptr) return dimensions end function unsafe_getdimensions( - mdarray::AbstractMDArray, -)::AbstractVector{<:AbstractDimension} + mdarray::AbstractMDArray{<:Any,D}, +)::NTuple{D,T where T<:AbstractDimension} where {D} @assert !isnull(mdarray) dimensionscountref = Ref{Csize_t}() dimensionshptr = GDAL.gdalmdarraygetdimensions(mdarray, dimensionscountref) - dimensions = AbstractDimension[ - Dimension(unsafe_load(dimensionshptr, n), mdarray.dataset) for - n in dimensionscountref[]:-1:1 - ] + dimensions = reverse( + ntuple( + d -> Dimension(unsafe_load(dimensionshptr, d), mdarray.dataset), + dimensionscountref[], + ), + ) GDAL.vsifree(dimensionshptr) return dimensions end -function Base.size(mdarray::AbstractMDArray) +function Base.size(mdarray::AbstractMDArray{<:Any,D})::NTuple{D,Int} where {D} getdimensions(mdarray) do dimensions - D = length(dimensions) return ntuple(d -> getsize(dimensions[d]), D) end end @@ -686,45 +697,41 @@ end Base.eltype(mdarray::AbstractMDArray{T}) where {T} = T -function getblocksize(mdarray::AbstractMDArray)::AbstractVector{Int64} +function getblocksize( + mdarray::AbstractMDArray{<:Any,D}, +)::NTuple{D,Int} where {D} @assert !isnull(mdarray) count = Ref{Csize_t}() blocksizeptr = GDAL.gdalmdarraygetblocksize(mdarray, count) - blocksize = Int64[unsafe_load(blocksizeptr, n) for n in count[]:-1:1] + blocksize = reverse(ntuple(d -> Int(unsafe_load(blocksizeptr, d)), count[])) GDAL.vsifree(blocksizeptr) return blocksize end DiskArrays.haschunks(::AbstractMDArray) = DiskArrays.Chunked() + function DiskArrays.eachchunk( mdarray::AbstractMDArray{<:Any,D}, )::NTuple{D,Int} where {D} blocksize = getblocksize(mdarray) - return DiskArrays.GridChunks(mdarray, Int.(blocksize)) + return DiskArrays.GridChunks(mdarray, blocksize) end function getprocessingchunksize( mdarray::AbstractMDArray, maxchunkmemory::Integer, -)::AbstractVector{Int64} +)::AbstractVector{Int} @assert !isnull(mdarray) count = Ref{Csize_t}() chunksizeptr = GDAL.gdalmdarraygetprocessingchunksize(mdarray, count, maxchunkmemory) - chunksize = Int64[unsafe_load(chunksizeptr, n) for n in count[]:-1:1] + chunksize = Int[unsafe_load(chunksizeptr, n) for n in count[]:-1:1] GDAL.vsifree(chunksizeptr) return chunksize end # processperchunk -const IndexLike{D} = - Union{AbstractVector{<:Integer},CartesianIndex{D},NTuple{D,<:Integer}} -const RangeLike{D} = Union{ - AbstractVector{<:AbstractRange{<:Integer}}, - NTuple{D,<:AbstractRange{<:Integer}}, -} - function read!( mdarray::AbstractMDArray, arraystartidx::IndexLike{D}, diff --git a/src/mdarray/types.jl b/src/mdarray/types.jl index 9b4a8276..ce26c6a9 100644 --- a/src/mdarray/types.jl +++ b/src/mdarray/types.jl @@ -176,6 +176,20 @@ isnull(x::StyleTool) = x.ptr == C_NULL ################################################################################ +const VectorLike{T} = Union{AbstractVector{<:T},NTuple{<:Any,X where X<:T}} + +const IndexLike{D} = Union{ + AbstractVector{<:Integer}, + CartesianIndex{D}, + NTuple{D,I where I<:Integer}, +} +const RangeLike{D} = Union{ + AbstractVector{<:AbstractRange{<:Integer}}, + NTuple{D,R where R<:AbstractRange{<:Integer}}, +} + +################################################################################ + Base.unsafe_convert(::Type{Ptr{Cvoid}}, x::AbstractExtendedDataType) = x.ptr Base.unsafe_convert(::Type{Ptr{Cvoid}}, x::AbstractEDTComponent) = x.ptr Base.unsafe_convert(::Type{Ptr{Cvoid}}, x::AbstractGroup) = x.ptr @@ -227,11 +241,11 @@ function destroy(dimension::AbstractDimension)::Nothing return nothing end -function destroy(edtcomponents::AbstractVector{<:AbstractEDTComponent}) +function destroy(edtcomponents::VectorLike{<:AbstractEDTComponent}) return destroy.(edtcomponents) end -destroy(attributes::AbstractVector{<:AbstractAttribute}) = destroy.(attributes) -destroy(dimensions::AbstractVector{<:AbstractDimension}) = destroy.(dimensions) +destroy(attributes::VectorLike{<:AbstractAttribute}) = destroy.(attributes) +destroy(dimensions::VectorLike{<:AbstractDimension}) = destroy.(dimensions) ################################################################################ @@ -280,6 +294,9 @@ struct DimensionHList return new(dimensionhs, dimensions) end end +function DimensionHList(dimensions::NTuple{<:Any,T where T<:AbstractDimension}) + return DimensionHList([dimensions...]) +end function Base.cconvert( ::Type{Ptr{GDAL.GDALDimensionH}}, diff --git a/test/test_mdarray.jl b/test/test_mdarray.jl index 102240d5..3f7a895b 100644 --- a/test/test_mdarray.jl +++ b/test/test_mdarray.jl @@ -153,7 +153,7 @@ end drivercreateoptions, ) @test !AG.isnull(dataset) - @test match(r"GDAL Dataset", string(dataset)) !== nothing + @test match(r"^GDAL Dataset", string(dataset)) !== nothing files = AG.filelist(dataset) if drivername in ["MEM"] @@ -166,7 +166,7 @@ end root = AG.getrootgroup(dataset) @test !AG.isnull(root) - @test match(r"ArchGDAL.Group", string(root)) !== nothing + @test match(r"^ArchGDAL.Group", string(root)) !== nothing rootname = AG.getname(root) @test rootname == "/" rootfullname = AG.getfullname(root) @@ -174,7 +174,7 @@ end group = AG.creategroup(root, "group") @test !AG.isnull(group) - @test match(r"ArchGDAL.IGroup", string(group)) !== nothing + @test match(r"^ArchGDAL.IGroup", string(group)) !== nothing @test AG.getname(group) == "group" @test AG.getfullname(group) == "/group" @@ -186,24 +186,41 @@ end nx, ny = 3, 4 dimx = AG.createdimension(group, "x", "", "", nx) @test !AG.isnull(dimx) - @test match(r"ArchGDAL.IDimension", string(dimx)) !== nothing + @test match(r"^ArchGDAL.IDimension", string(dimx)) !== nothing dimy = AG.createdimension(group, "y", "", "", ny) @test !AG.isnull(dimy) datatype = AG.extendeddatatypecreate(Float32) @test !AG.isnull(datatype) - @test match(r"ArchGDAL.IExtendedDataType", string(datatype)) !== - nothing + @test match( + r"^ArchGDAL.IExtendedDataType", + string(datatype), + ) !== nothing + + if drivername != "netCDF" + # netCDF does not support deleting MDArrays + mdarray0 = AG.createmdarray( + group, + "mdarray0", + [dimx, dimy], + datatype, + mdarraycreateoptions, + ) + @test !AG.isnull(mdarray0) + success = AG.deletemdarray(group, "mdarray0") + @test success + end mdarray = AG.createmdarray( group, "mdarray", - [dimx, dimy], + (dimx, dimy), datatype, mdarraycreateoptions, ) @test !AG.isnull(mdarray) - @test match(r"ArchGDAL.IMDArray", string(mdarray)) !== nothing + @test match(r"^3×4 ArchGDAL.IMDArray", string(mdarray)) !== + nothing @test AG.getmdarraynames(root) == [] @test AG.getmdarraynames(group) == ["mdarray"] @@ -385,10 +402,28 @@ end ) do datatype @test !AG.isnull(datatype) + if drivername != "netCDF" + # netCDF does not support deleting MDArrays + AG.createmdarray( + group, + "mdarray0", + [dimx, dimy], + datatype, + mdarraycreateoptions, + ) do mdarray0 + @test !AG.isnull(mdarray0) + end + success = AG.deletemdarray( + group, + "mdarray0", + ) + @test success + end + AG.createmdarray( group, "mdarray", - [dimx, dimy], + (dimx, dimy), datatype, mdarraycreateoptions, ) do mdarray From 9324fd16acbc9d3bfa2e51d4867b0dd323ffee4e Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 21 Aug 2024 11:22:36 -0400 Subject: [PATCH 19/19] Correct indices in MDArray.adviseread --- src/mdarray/mdarray.jl | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/mdarray/mdarray.jl b/src/mdarray/mdarray.jl index 1231607d..ffee5447 100644 --- a/src/mdarray/mdarray.jl +++ b/src/mdarray/mdarray.jl @@ -568,10 +568,14 @@ function adviseread( @assert !isnull(mdarray) @assert isnothing(arraystartix) ? true : length(arraystartidx) == D @assert isnothing(count) ? true : length(count == D) + gdal_arraystartidx = + isnothing(arraystartidx) ? C_NULL : + UInt64[arraystartidx[d] - 1 for d in D:-1:1] + gdal_count = isnothing(count) ? C_NULL : Csize_t[count[d] for d in D:-1:1] return GDAL.gdalmdarrayadviseread( mdarray, - isnothing(arraystartidx) ? C_NULL : reverse(arraystartidx), - isnothing(count) ? C_NULL : count, + gdal_arraystartidx, + gdal_count, CSLConstListWrapper(options), ) end @@ -743,11 +747,11 @@ function read!( @assert length(arraystartidx) == D @assert length(count) == D @assert isnothing(arraystep) ? true : length(arraystep) == D - gdal_arraystartidx = UInt64[arraystartidx[n] - 1 for n in D:-1:1] - gdal_count = Csize_t[count[n] for n in D:-1:1] + gdal_arraystartidx = UInt64[arraystartidx[d] - 1 for d in D:-1:1] + gdal_count = Csize_t[count[d] for d in D:-1:1] gdal_arraystep = - isnothing(arraystep) ? nothing : Int64[arraystep[n] for n in D:-1:1] - gdal_bufferstride = Cptrdiff_t[stride(buffer, n) for n in D:-1:1] + isnothing(arraystep) ? C_NULL : Int64[arraystep[d] for d in D:-1:1] + gdal_bufferstride = Cptrdiff_t[stride(buffer, d) for d in D:-1:1] return extendeddatatypecreate(T) do bufferdatatype success = GDAL.gdalmdarrayread( mdarray, @@ -840,11 +844,11 @@ function write( @assert length(arraystartidx) == D @assert length(count) == D @assert isnothing(arraystep) ? true : length(arraystep) == D - gdal_arraystartidx = UInt64[arraystartidx[n] - 1 for n in D:-1:1] - gdal_count = Csize_t[count[n] for n in D:-1:1] + gdal_arraystartidx = UInt64[arraystartidx[d] - 1 for d in D:-1:1] + gdal_count = Csize_t[count[d] for d in D:-1:1] gdal_arraystep = - isnothing(arraystep) ? nothing : Int64[arraystep[n] for n in D:-1:1] - gdal_bufferstride = Cptrdiff_t[stride(buffer, n) for n in D:-1:1] + isnothing(arraystep) ? C_NULL : Int64[arraystep[d] for d in D:-1:1] + gdal_bufferstride = Cptrdiff_t[stride(buffer, d) for d in D:-1:1] return extendeddatatypecreate(T) do bufferdatatype success = GDAL.gdalmdarraywrite( mdarray,