From f019c6341272fdf40e5bf08c32e84a0ad407ef63 Mon Sep 17 00:00:00 2001 From: Ognyan Moore Date: Fri, 22 Sep 2023 12:19:57 -0500 Subject: [PATCH 1/2] Update script for FOSS4G 2023 NA Image Creation --- workshop/create_iamge.sh | 136 ++++++++++++++++++++++++++++++++ workshop/docker/Dockerfile | 22 ++++++ workshop/docker/environment.yml | 12 +++ workshop/workshop_zip.sh | 69 ---------------- 4 files changed, 170 insertions(+), 69 deletions(-) create mode 100644 workshop/create_iamge.sh create mode 100644 workshop/docker/Dockerfile create mode 100644 workshop/docker/environment.yml delete mode 100644 workshop/workshop_zip.sh diff --git a/workshop/create_iamge.sh b/workshop/create_iamge.sh new file mode 100644 index 0000000..5311fb4 --- /dev/null +++ b/workshop/create_iamge.sh @@ -0,0 +1,136 @@ +eval "$(conda shell.bash hook)" + +export ROOT_DIR="$PWD/PDAL Workshop Materials" +# create root_directory for image +mkdir -p "$ROOT_DIR" + +# grab PDAL repo +git clone -n --depth=1 https://github.com/PDAL/PDAL + +# build pdal docs +conda create -n "pdal-docs" python=3.11 --yes --quiet +conda activate pdal-docs +mamba install -c conda-forge graphviz --yes --quiet +pip install -r PDAL/doc/requirements.txt +make -C PDAL/doc doxygen +make -C PDAL/doc html +make -C PDAL/doc latexpdf + +# copy workshop docs to image directory +cp -R PDAL/doc/build/html "$ROOT_DIR/docs" +cp PDAL/doc/build/latex/PDAL.pdf "$ROOT_DIR"/docs + +# copy needed datasets +git clone --depth=1 https://github.com/PDAL/data +cd data +git lfs install +git lfs pull +cd .. + +mkdir -p "$ROOT_DIR/exercises/batch_processing" +cp PDAL/doc/workshop/generation/batch_processing/batch_srs_gdal.json \ + "$ROOT_DIR/exercises/batch_processing" + + +mkdir -p "$ROOT_DIR/exercises/batch_processing/source" +cp data/workshop/TM_551_101.laz \ + data/workshop/TM_551_102.laz \ + data/workshop/TM_552_101.laz \ + data/workshop/TM_552_102.laz \ + "$ROOT_DIR/exercises/batch_processing/source/" + +mkdir -p "$ROOT_DIR/exercises/analysis/clipping" +cp data/workshop/autzen.laz \ + PDAL/doc/workshop/manipulation/clipping/clipping.json \ + "$ROOT_DIR/exercises/analysis/clipping/" + + +mkdir -p "$ROOT_DIR/exercises/analysis/colorization" +cp data/workshop/casi-2015-04-29-weekly-mosaic.tif \ + PDAL/doc/workshop/manipulation/colorization/colorize.json \ + "$ROOT_DIR/exercises/analysis/colorization/" + +mkdir -p "$ROOT_DIR/exercises/analysis/denoising" +cp PDAL/doc/workshop/manipulation/denoising/denoise.json \ + data/workshop/18TWK820985.laz \ + "$ROOT_DIR/exercises/analysis/denoising/" + +mkdir -p "$ROOT_DIR/exercises/analysis/density" +cp data/workshop/uncompahgre.laz "$ROOT_DIR/exercises/analysis/density/" + +mkdir -p "$ROOT_DIR/exercises/analysis/dtm" +cp PDAL/doc/workshop/generation/dtm/gdal.json "$ROOT_DIR/exercises/analysis/dtm" + +mkdir -p "$ROOT_DIR/exercises/analysis/ground" +cp data/workshop/CSite1_orig-utm.laz "$ROOT_DIR/exercises/analysis/ground/" + +mkdir -p "$ROOT_DIR/exercises/analysis/rasterize" +cp PDAL/doc/workshop/generation/rasterize/classification.json "$ROOT_DIR/exercises/analysis/rasterize" + +mkdir -p "$ROOT_DIR/exercises/analysis/thinning" +cp data/workshop/uncompahgre.laz \ + data/workshop/uncompahgre.copc.laz \ + "$ROOT_DIR/exercises/analysis/thinning/" + +mkdir -p "$ROOT_DIR/exercises/georeferencing" +cp data/workshop/S1C1_csd_004.csd "$ROOT_DIR/exercises/georeferencing" + +mkdir -p "$ROOT_DIR/exercises/python" +cp PDAL/doc/workshop/generation/python/histogram.json \ + PDAL/doc/workshop/generation/python/histogram.py \ + data/workshop/athletic-fields.laz \ + "$ROOT_DIR/exercises/python/" + +mkdir -p "$ROOT_DIR/exercises/translation" +cp data/workshop/csite-dd.laz \ + data/workshop/interesting.laz \ + PDAL/doc/workshop/introduction/entwine.json \ + "$ROOT_DIR/exercises/translation" + +cp data/workshop/interesting.las "$ROOT_DIR/exercises/translation" + +mkdir -p "$ROOT_DIR/exercises/info" +cp data/workshop/interesting.las "$ROOT_DIR/exercises/info/" + +curl --output-dir "$ROOT_DIR/software/windows" -C - -OL --create-dirs https://download.qgis.org/downloads/QGIS-OSGeo4W-3.32.2-1.msi +curl --output-dir "$ROOT_DIR/software/windows" -C - -OL --create-dirs https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Windows-x86_64.exe +curl --output-dir "$ROOT_DIR/software/windows" -C - -OL --create-dirs https://github.com/jqlang/jq/releases/download/jq-1.7/jq-windows-amd64.exe +curl --output-dir "$ROOT_DIR/software/windows" -C - -OL --create-dirs https://www.danielgm.net/cc/release/CloudCompare_v2.13.beta_setup_x64.exe + +# grab macOS Installers +curl --output-dir "$ROOT_DIR/software/macOS" -C - -OL --create-dirs https://download.qgis.org/downloads/macos/qgis-macos-pr.dmg +curl --output-dir "$ROOT_DIR/software/macOS" -C - -OL --create-dirs https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-MacOSX-arm64.sh +curl --output-dir "$ROOT_DIR/software/macOS" -C - -OL --create-dirs https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-MacOSX-x86_64.sh +curl --output-dir "$ROOT_DIR/software/macOS" -C - -OL --create-dirs https://github.com/jqlang/jq/releases/download/jq-1.7/jq-macos-amd64 +curl --output-dir "$ROOT_DIR/software/macOS" -C - -OL --create-dirs https://github.com/jqlang/jq/releases/download/jq-1.7/jq-macos-arm64 +curl --output-dir "$ROOT_DIR/software/macOS" -C - -OL --create-dirs https://www.danielgm.net/cc/release/CloudCompare-2.13.0-x86_64.dmg +curl --output-dir "$ROOT_DIR/software/macOS" -C - -OL --create-dirs https://www.danielgm.net/cc/release/CloudCompare-2.13.0-arm64.dmg + +# mamba env create -f environment-pdal_workshop_image.yml --yes --quiet +conda create -n "pdal-gen-workshop-image" python=3.11 --yes --quiet +conda activate pdal-gen-workshop-image +mamba install -c conda-forge conda-pack awscli --yes --quiet + +# grab cool-lidar content +aws s3 sync s3://cool-lidar "$ROOT_DIR/cool-lidar" + +# grab conda environments +mkdir -p "$ROOT_DIR/software/conda_environments" + +# make docker environment +# NOTE: need to do this for x86_64 +mkdir docker +cd docker +docker image build -t pdal-workshop data/workshop/docker +docker save -o pdal-workshop_docker-arm64.tar.gz pdal-workshop +cd .. +cp "docker/pdal-workshop_docker*.tar.gz" "$ROOT_DIR/software/conda_environments/" + + +mkdir -p staging/conda_environments +# NOTE: this needs to happen for every platform, not just osx-arm64 +mamba env create --file "$PWD/PDAL/doc/workshop/environment.yml" -p "$PWD/staging/conda_environments" --yes --quiet +conda-pack -p staging/conda_environments -o "$ROOT_DIR/software/conda_environments/pdal-workshop_osx-arm64.tar.gz" -f + + + diff --git a/workshop/docker/Dockerfile b/workshop/docker/Dockerfile new file mode 100644 index 0000000..e7d64a5 --- /dev/null +++ b/workshop/docker/Dockerfile @@ -0,0 +1,22 @@ +FROM condaforge/mambaforge:latest as build +MAINTAINER Howard Butler + +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 + +COPY environment.yml . +RUN mamba env create -f environment.yml + +RUN mamba install -c conda-forge conda-pack + +RUN conda-pack -n "pdal-workshop" -o /tmp/env.tar && \ + mkdir /venv && cd /venv && tar xf /tmp/env.tar && \ + rm /tmp/env.tar + +RUN /venv/bin/conda-unpack + +FROM debian:bookworm-slim AS runtime + +COPY --from=build /venv /venv + +SHELL ["/bin/bash", "-c"] +ENTRYPOINT source /venv/bin/activate \ No newline at end of file diff --git a/workshop/docker/environment.yml b/workshop/docker/environment.yml new file mode 100644 index 0000000..5eeea16 --- /dev/null +++ b/workshop/docker/environment.yml @@ -0,0 +1,12 @@ +name: pdal-workshop +channels: + - conda-forge +dependencies: + - python=3.11 + - pdal=2.5.6 + - python-pdal + - gdal + - untwine + - geopandas + - jq + diff --git a/workshop/workshop_zip.sh b/workshop/workshop_zip.sh deleted file mode 100644 index 90564d7..0000000 --- a/workshop/workshop_zip.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -# curl https://pdal.io/PDAL.pdf -o PDAL-docs.pdf - -git clone -n --depth=1 --filter=tree:0 https://github.com/PDAL/PDAL -cd PDAL -git sparse-checkout set --no-cone doc -git checkout -cd .. - -git clone -n --depth=1 --filter=tree:0 https://github.com/PDAL/data -cd data -git sparse-checkout set --no-cone workshop -git checkout -cd .. - -mkdir -p exercises exercises/info -mkdir -p software software/MacOS software/Windows software/Linux - -# Moving docs -mv PDAL/doc/workshop/manipulation exercises -mv exercises/manipulation exercises/analysis -mv PDAL/doc/workshop/generation/meshing PDAL/doc/workshop/generation/rasterize \ - PDAL/doc/workshop/generation/dtm exercises/analysis -mv PDAL/doc/workshop/generation/batch_processing PDAL/doc/workshop/generation/georeferencing \ - PDAL/doc/workshop/generation/python PDAL/doc/workshop/introduction exercises -mv exercises/introduction exercises/translation -mv exercises/translation/metadata.rst exercises/translation/single-point.rst \ - exercises/translation/near.rst exercises/info -# Moving data -mkdir -p exercises/batch_processing/source -mv data/workshop/TM_551_101.laz data/workshop/TM_551_102.laz data/workshop/TM_552_101.laz \ - data/workshop/TM_552_102.laz exercises/batch_processing/source -mv data/workshop/autzen.laz exercises/analysis/clipping -mv data/workshop/casi-2015-04-29-weekly-mosaic.tif exercises/analysis/colorization -mv data/workshop/18TWK820985.laz exercises/analysis/denoising -cp data/workshop/uncompahgre.laz exercises/analysis/density -mv data/workshop/CSite1_orig-utm.laz exercises/analysis/ground -mv data/workshop/uncompahgre.laz data/workshop/uncompahgre.copc.laz exercises/analysis/thinning -mv data/workshop/S1C1_csd_004.csd exercises/georeferencing -mv data/workshop/athletic-fields.laz exercises/python -mv data/workshop/csite-dd.laz data/workshop/interesting.laz exercises/translation -cp data/workshop/interesting.las exercises/translation -mv data/workshop/interesting.las exercises/info - -# software download -cd software/Windows -curl https://qgis.org/downloads/QGIS-OSGeo4W-3.32.0-1.msi -o QGIS-OSGeo4W-3.32.0-1.msi -curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -o Miniconda3-latest-Windows-x86_64.exe -curl https://www.danielgm.net/cc/release/CloudCompare_v2.13.alpha_setup_x64.exe -o CloudCompare_v2.13.alpha_setup_x64.exe -cd .. -cd MacOS -curl https://qgis.org/downloads/macos/qgis-macos-pr.dmg -o qgis-macos-pr.dmg -curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o Miniconda3-latest-MacOSX-x86_64.sh -curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o Miniconda3-latest-MacOSX-arm64.sh -curl https://www.danielgm.net/cc/release/CloudCompare-2.13.0-x86_64.dmg -o CloudCompare-2.13.0-x86_64.dmg -curl https://www.danielgm.net/cc/release/CloudCompare-2.13.0-arm64.dmg -o CloudCompare-2.13.0-arm64.dmg -cd .. -cd Linux -curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o Miniconda3-latest-Linux-x86_64.sh -cd ../.. - -chmod -R +w data PDAL exercises software -rm -rf data PDAL - -zip_filename="PDAL_Workshop_complete.zip" -zip -r "$zip_filename" exercises software #PDAL-docs.pdf - -rm -rf exercises software #PDAL-docs.pdf \ No newline at end of file From 68342960cfa9fd401028b793fa9c7f70504683c7 Mon Sep 17 00:00:00 2001 From: Ognyan Moore Date: Thu, 14 Dec 2023 14:57:48 -0800 Subject: [PATCH 2/2] Small fixes, include missing files --- workshop/{create_iamge.sh => create_image.sh} | 25 ++++++++----------- workshop/docker/Dockerfile | 4 +-- workshop/docker/environment.yml | 2 +- 3 files changed, 14 insertions(+), 17 deletions(-) rename workshop/{create_iamge.sh => create_image.sh} (87%) diff --git a/workshop/create_iamge.sh b/workshop/create_image.sh similarity index 87% rename from workshop/create_iamge.sh rename to workshop/create_image.sh index 5311fb4..8dff18b 100644 --- a/workshop/create_iamge.sh +++ b/workshop/create_image.sh @@ -5,7 +5,7 @@ export ROOT_DIR="$PWD/PDAL Workshop Materials" mkdir -p "$ROOT_DIR" # grab PDAL repo -git clone -n --depth=1 https://github.com/PDAL/PDAL +git clone --depth=1 https://github.com/PDAL/PDAL # build pdal docs conda create -n "pdal-docs" python=3.11 --yes --quiet @@ -42,6 +42,8 @@ cp data/workshop/TM_551_101.laz \ mkdir -p "$ROOT_DIR/exercises/analysis/clipping" cp data/workshop/autzen.laz \ PDAL/doc/workshop/manipulation/clipping/clipping.json \ + PDAL/doc/workshop/manipulation/clipping/attributes.json \ + PDAL/doc/workshop/manipulation/clipping/attributes.vrt \ "$ROOT_DIR/exercises/analysis/clipping/" @@ -92,15 +94,15 @@ cp data/workshop/interesting.las "$ROOT_DIR/exercises/translation" mkdir -p "$ROOT_DIR/exercises/info" cp data/workshop/interesting.las "$ROOT_DIR/exercises/info/" -curl --output-dir "$ROOT_DIR/software/windows" -C - -OL --create-dirs https://download.qgis.org/downloads/QGIS-OSGeo4W-3.32.2-1.msi -curl --output-dir "$ROOT_DIR/software/windows" -C - -OL --create-dirs https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Windows-x86_64.exe +curl --output-dir "$ROOT_DIR/software/windows" -C - -OL --create-dirs https://qgis.org/downloads/QGIS-OSGeo4W-3.34.1-2.msi +curl --output-dir "$ROOT_DIR/software/windows" -C - -OL --create-dirs https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Windows-x86_64.exe curl --output-dir "$ROOT_DIR/software/windows" -C - -OL --create-dirs https://github.com/jqlang/jq/releases/download/jq-1.7/jq-windows-amd64.exe curl --output-dir "$ROOT_DIR/software/windows" -C - -OL --create-dirs https://www.danielgm.net/cc/release/CloudCompare_v2.13.beta_setup_x64.exe # grab macOS Installers curl --output-dir "$ROOT_DIR/software/macOS" -C - -OL --create-dirs https://download.qgis.org/downloads/macos/qgis-macos-pr.dmg -curl --output-dir "$ROOT_DIR/software/macOS" -C - -OL --create-dirs https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-MacOSX-arm64.sh -curl --output-dir "$ROOT_DIR/software/macOS" -C - -OL --create-dirs https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-MacOSX-x86_64.sh +curl --output-dir "$ROOT_DIR/software/macOS" -C - -OL --create-dirs https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-arm64.sh +curl --output-dir "$ROOT_DIR/software/macOS" -C - -OL --create-dirs https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-x86_64.sh curl --output-dir "$ROOT_DIR/software/macOS" -C - -OL --create-dirs https://github.com/jqlang/jq/releases/download/jq-1.7/jq-macos-amd64 curl --output-dir "$ROOT_DIR/software/macOS" -C - -OL --create-dirs https://github.com/jqlang/jq/releases/download/jq-1.7/jq-macos-arm64 curl --output-dir "$ROOT_DIR/software/macOS" -C - -OL --create-dirs https://www.danielgm.net/cc/release/CloudCompare-2.13.0-x86_64.dmg @@ -118,19 +120,14 @@ aws s3 sync s3://cool-lidar "$ROOT_DIR/cool-lidar" mkdir -p "$ROOT_DIR/software/conda_environments" # make docker environment -# NOTE: need to do this for x86_64 -mkdir docker -cd docker +# NOTE: need to do this for x86_64 as well! docker image build -t pdal-workshop data/workshop/docker -docker save -o pdal-workshop_docker-arm64.tar.gz pdal-workshop -cd .. -cp "docker/pdal-workshop_docker*.tar.gz" "$ROOT_DIR/software/conda_environments/" - +docker save -o "$ROOT_DIR/software/conda_environments/pdal-workshop_docker-arm64.tar.gz" pdal-workshop mkdir -p staging/conda_environments # NOTE: this needs to happen for every platform, not just osx-arm64 mamba env create --file "$PWD/PDAL/doc/workshop/environment.yml" -p "$PWD/staging/conda_environments" --yes --quiet conda-pack -p staging/conda_environments -o "$ROOT_DIR/software/conda_environments/pdal-workshop_osx-arm64.tar.gz" -f - - +CONDA_SUBDIR=osx-64 mamba env create --file "$PWD/PDAL/doc/workshop/environment.yml" -p "$PWD/staging/conda_environments" --yes --quiet +conda-pack -p staging/conda_environments -o "$ROOT_DIR/software/conda_environments/pdal-workshop_osx-x86_64.tar.gz" -f diff --git a/workshop/docker/Dockerfile b/workshop/docker/Dockerfile index e7d64a5..8f9e8d1 100644 --- a/workshop/docker/Dockerfile +++ b/workshop/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM condaforge/mambaforge:latest as build +FROM condaforge/miniforge3:latest as build MAINTAINER Howard Butler ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 @@ -19,4 +19,4 @@ FROM debian:bookworm-slim AS runtime COPY --from=build /venv /venv SHELL ["/bin/bash", "-c"] -ENTRYPOINT source /venv/bin/activate \ No newline at end of file +ENTRYPOINT source /venv/bin/activate diff --git a/workshop/docker/environment.yml b/workshop/docker/environment.yml index 5eeea16..0f60139 100644 --- a/workshop/docker/environment.yml +++ b/workshop/docker/environment.yml @@ -3,7 +3,7 @@ channels: - conda-forge dependencies: - python=3.11 - - pdal=2.5.6 + - pdal=2.6.2 - python-pdal - gdal - untwine