Adds an experimental byod-raster-import command... #5614
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Build | |
| on: [push, pull_request] | |
| env: | |
| PY_VER: "3.10" | |
| CMAKE_VERSION: "~3.25.0" | |
| FORCE_COLOR: "YES" | |
| KART_S3_TEST_DATA_POINT_CLOUD: "s3://kart-bring-your-own-data-poc/auckland-small-laz1.2/*.laz" | |
| KART_S3_TEST_DATA_RASTER: "s3://kart-bring-your-own-data-poc/erorisk_si/*.tif" | |
| AWS_NO_SIGN_REQUEST: "1" | |
| jobs: | |
| # | |
| # Linux development build on modern Ubuntu | |
| # | |
| Linux-dev: | |
| name: "Build Linux dev" | |
| runs-on: ubuntu-22.04 | |
| # We want to run on external PRs, but not on our own internal PRs as they'll be run | |
| # by the push to the branch. | |
| # https://github.community/t/duplicate-checks-on-push-and-pull-request-simultaneous-event/18012/7 | |
| if: > | |
| ( | |
| github.event_name == 'push' | |
| || github.event.pull_request.head.repo.full_name != github.repository | |
| ) | |
| env: | |
| CCACHE_DIR: ${{ github.workspace }}/.cache/ccache | |
| CCACHE_COMPRESS: "1" | |
| KART_POSTGIS_URL: "postgresql://postgres:@localhost:5432/postgres" | |
| KART_SQLSERVER_URL: "mssql://sa:PassWord1@localhost:1433/master" | |
| KART_MYSQL_URL: "mysql://root:PassWord1@localhost:3306" | |
| services: | |
| postgis: | |
| # https://github.com/postgis/docker-postgis/issues/216 | |
| image: ghcr.io/baosystems/postgis:15 | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 2s | |
| --health-timeout 2s | |
| --health-retries 5 | |
| -e POSTGRES_HOST_AUTH_METHOD=trust | |
| ports: | |
| - 5432:5432 | |
| sqlserver: | |
| image: mcr.microsoft.com/mssql/server | |
| options: >- | |
| -e ACCEPT_EULA=Y | |
| -e SA_PASSWORD=PassWord1 | |
| ports: | |
| - 1433:1433 | |
| mysql: | |
| image: mysql | |
| options: >- | |
| -e MYSQL_ROOT_PASSWORD=PassWord1 | |
| ports: | |
| - 3306:3306 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| # | |
| # setup | |
| # | |
| - name: "setup: cmake & ninja" | |
| uses: lukka/get-cmake@arm64-win-linux | |
| with: | |
| cmakeVersion: "${{ env.CMAKE_VERSION }}" | |
| - name: "setup: ccache" | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ env.CCACHE_DIR }} | |
| key: vendor-ccache-ubuntu-22.04-cmake-vcpkg | |
| - name: "setup: misc" | |
| run: | | |
| sudo apt-get update -q -y | |
| sudo apt-get install -q -y --no-install-recommends ccache | |
| mkdir -p ${{ env.CCACHE_DIR }} | |
| echo "/usr/lib/ccache" >> $GITHUB_PATH | |
| - name: "setup: vcpkg" | |
| uses: lukka/run-vcpkg@v10 | |
| with: | |
| vcpkgDirectory: "${{ github.workspace }}/vcpkg-vendor/vcpkg" | |
| vcpkgJsonGlob: "**/vcpkg-vendor/vcpkg.json" | |
| prependedCacheKey: "20221110a-ubuntu-22.04" | |
| appendedCacheKey: "C${{ env.CMAKE_VERSION}}:${{ hashFiles('vcpkg-vendor/vcpkg-overlay-triplets/**', 'vcpkg-vendor/vcpkg-overlay-ports/**', 'vcpkg-vendor/vcpkg-toolchain-setup.cmake') }}" | |
| # | |
| # App Build | |
| # | |
| - name: "app: version" | |
| id: version | |
| run: | | |
| if [[ '${{ github.ref_type }}' == 'tag' ]] && [[ '${{ github.ref_name }}' =~ ^v(.*) ]]; then | |
| VER=$(echo ${{ github.ref_name }} | sed 's/^v//') | |
| IS_RELEASE=1 | |
| else | |
| VER=$(sed -E "s/(.*)/\1+ci.${{ github.run_number }}.git${GITHUB_SHA::8}/" kart/VERSION) | |
| IS_RELEASE=0 | |
| fi | |
| echo "App Version: $VER" | |
| echo "Is Release? $IS_RELEASE" | |
| echo "value=$VER" >> $GITHUB_OUTPUT | |
| echo "KART_VERSION=$VER" >> $GITHUB_ENV | |
| echo "is_release=$IS_RELEASE" >> $GITHUB_OUTPUT | |
| - name: "app: configuration & vendor dependencies" | |
| uses: lukka/run-cmake@v10 | |
| with: | |
| configurePreset: ci-linux | |
| - name: "check for vcpkg errors" | |
| if: "failure() || cancelled()" | |
| run: | | |
| echo "RUNVCPKG_NO_CACHE=1" >> $GITHUB_ENV | |
| - name: "app: build" | |
| uses: lukka/run-cmake@v10 | |
| with: | |
| configurePreset: ci-linux | |
| buildPreset: ci-linux | |
| # | |
| # App tests & checks | |
| # | |
| - name: "test: smoke test" | |
| run: | | |
| ./build/kart --version | |
| - name: "test: create no_postgis environment for tests" | |
| shell: bash | |
| run: | | |
| sudo apt-get -y install postgresql | |
| psql postgresql://postgres:@localhost:5432/postgres -c 'CREATE DATABASE no_postgis;' | |
| echo "KART_NO_POSTGIS_URL=postgresql://postgres:@localhost:5432/no_postgis" >> $GITHUB_ENV | |
| - name: "test: install database drivers for tests" | |
| run: | | |
| curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - | |
| curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list > /dev/null | |
| sudo apt-get update -q -y | |
| sudo ACCEPT_EULA=Y apt-get install -q -y msodbcsql17 | |
| - name: "test: install fienode for reflink testing" | |
| run: | | |
| curl -L https://github.com/pwaller/fienode/releases/download/v1.0/fienode-linux-amd64 --output fienode | |
| chmod +x fienode | |
| mv fienode /usr/local/bin/ | |
| - name: "test: unit tests" | |
| uses: lukka/run-cmake@v10 | |
| with: | |
| configurePreset: ci-linux | |
| testPreset: ci-linux | |
| - name: "test: dependency license check" | |
| run: | | |
| cmake --build build --target py-license-check | |
| # | |
| # Packaging | |
| # | |
| - name: "bundle: assemble" | |
| uses: lukka/run-cmake@v10 | |
| with: | |
| configurePreset: ci-linux | |
| buildPreset: ci-bundle-linux | |
| - name: "bundle: smoke test" | |
| shell: bash | |
| run: | | |
| ./build/pyinstaller/dist/kart/kart --version | |
| - name: "bundle: e2e tests" | |
| uses: lukka/run-cmake@v10 | |
| with: | |
| configurePreset: ci-linux | |
| testPreset: ci-e2e-linux | |
| # | |
| # Install | |
| # | |
| - name: "install" | |
| shell: bash | |
| run: | | |
| cmake --install ./build --verbose | |
| /opt/kart/kart --version | |
| command -v kart | |
| kart --version | |
| - name: "install: e2e tests" | |
| shell: bash | |
| run: | | |
| time tests/scripts/e2e-1.sh | |
| # | |
| # Linux build for installers | |
| # Runs on centos 7 for maximum compatibility | |
| # | |
| Linux-Package: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - base: ubuntu-22.04 | |
| label: centos7/amd64 | |
| image: quay.io/pypa/manylinux2014_x86_64 | |
| arch: amd64 | |
| arch_triplet: x64-linux | |
| - base: buildjet-4vcpu-ubuntu-2204-arm | |
| label: centos7/arm64 | |
| image: quay.io/pypa/manylinux2014_aarch64 | |
| arch: arm64 | |
| arch_triplet: arm64-linux | |
| name: Build Linux package ${{ matrix.os.arch }} | |
| runs-on: ${{ matrix.os.base }} | |
| # We want to run on external PRs, but not on our own internal PRs as they'll be run | |
| # by the push to the branch. | |
| # https://github.community/t/duplicate-checks-on-push-and-pull-request-simultaneous-event/18012/7 | |
| if: > | |
| ( | |
| github.event_name == 'push' | |
| || github.event.pull_request.head.repo.full_name != github.repository | |
| ) | |
| services: | |
| postgis: | |
| # https://github.com/postgis/docker-postgis/issues/216 | |
| image: ghcr.io/baosystems/postgis:15 | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 2s | |
| --health-timeout 2s | |
| --health-retries 5 | |
| -e POSTGRES_HOST_AUTH_METHOD=trust | |
| ports: | |
| - 5432:5432 | |
| sqlserver: | |
| image: mcr.microsoft.com/mssql/server | |
| options: >- | |
| -e ACCEPT_EULA=Y | |
| -e SA_PASSWORD=PassWord1 | |
| ports: | |
| - 1433:1433 | |
| mysql: | |
| image: mysql | |
| options: >- | |
| -e MYSQL_ROOT_PASSWORD=PassWord1 | |
| ports: | |
| - 3306:3306 | |
| env: | |
| KART_POSTGIS_URL: "postgresql://postgres:@postgis:5432/postgres" | |
| SQLSERVER_URL: "mssql://sa:PassWord1@sqlserver:1433/master?TrustServerCertificate=yes" | |
| KART_MYSQL_URL: "mysql://root:PassWord1@mysql:3306" | |
| ARCH: "${{ matrix.os.arch }}" | |
| ARCH_TRIPLET: "${{ matrix.os.arch_triplet }}" | |
| GITHUB_WORKSPACE: "/__w/kart/kart" # FIXME? | |
| PKG_CONFIG: "/__w/kart/kart/vcpkg-vendor/vcpkg/installed/${{ matrix.os.arch_triplet }}/tools/pkgconf/pkgconf" | |
| PKG_CONFIG_PATH: "/__w/kart/kart/vcpkg-vendor/vcpkg/installed/${{ matrix.os.arch_triplet }}/lib/pkgconfig" | |
| OPTIONS_FOR_CMAKE: >- | |
| [ | |
| '-DPKG_CONFIG_EXECUTABLE=/__w/kart/kart/vcpkg-vendor/vcpkg/installed/${{ matrix.os.arch_triplet }}/tools/pkgconf/pkgconf', | |
| '-DCPACK_DEBIAN_PACKAGE_ARCHITECTURE=${{ matrix.os.arch }}' | |
| ] | |
| container: | |
| image: ${{ matrix.os.image }} | |
| env: | |
| ACLOCAL_PATH: "/usr/local/share/aclocal:/usr/share/aclocal" | |
| PATH: "/opt/python/cp310-cp310/bin:/opt/rh/devtoolset-10/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/github/home/.cargo/bin:/opt/mssql-tools/bin" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| # | |
| # setup | |
| # | |
| - name: "setup: cmake & ninja" | |
| uses: lukka/get-cmake@arm64-win-linux | |
| with: | |
| cmakeVersion: "${{ env.CMAKE_VERSION }}" | |
| - name: "setup: rust" | |
| uses: ATiltedTree/setup-rust@v1 | |
| with: | |
| rust-version: stable | |
| - name: "setup: misc" | |
| shell: bash | |
| run: | | |
| echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV | |
| git config --global --add safe.directory ${GITHUB_WORKSPACE} | |
| yum install -y perl-IPC-Cmd rpm-build unixODBC zip | |
| echo "installing golang..." | |
| curl -qL "https://go.dev/dl/go1.19.2.linux-${ARCH}.tar.gz" | tar xz -C /usr/local/ | |
| ln -sf /usr/local/go/bin/go /usr/local/go/bin/gofmt /usr/local/bin/ | |
| if [ "${ARCH}" == "arm64" ]; then | |
| echo "VCPKG_FORCE_SYSTEM_BINARIES=1" >> $GITHUB_ENV | |
| fi | |
| - name: "setup: vcpkg" | |
| uses: lukka/run-vcpkg@v10 | |
| with: | |
| vcpkgDirectory: "vcpkg-vendor/vcpkg" | |
| vcpkgJsonGlob: "**/vcpkg-vendor/vcpkg.json" | |
| prependedCacheKey: "20230113a-linux-${{ matrix.os.label }}" | |
| appendedCacheKey: "C${{ env.CMAKE_VERSION}}:${{ hashFiles('vcpkg-vendor/vcpkg-overlay-triplets/**', 'vcpkg-vendor/vcpkg-overlay-ports/**', 'vcpkg-vendor/vcpkg-toolchain-setup.cmake') }}" | |
| - name: "setup: pkgconf" | |
| working-directory: /__w/_temp | |
| shell: bash | |
| run: | | |
| ${GITHUB_WORKSPACE}/vcpkg-vendor/vcpkg/vcpkg install pkgconf | |
| # | |
| # App Build | |
| # | |
| - name: "app: version" | |
| id: version | |
| shell: bash | |
| run: | | |
| if [[ '${{ github.ref_type }}' == 'tag' ]] && [[ '${{ github.ref_name }}' =~ ^v(.*) ]]; then | |
| VER=$(echo ${{ github.ref_name }} | sed 's/^v//') | |
| IS_RELEASE=1 | |
| else | |
| VER=$(sed -E "s/(.*)/\1+ci.${{ github.run_number }}.git${GITHUB_SHA::8}/" kart/VERSION) | |
| IS_RELEASE=0 | |
| fi | |
| echo "App Version: $VER" | |
| echo "Is Release? $IS_RELEASE" | |
| echo "value=$VER" >> $GITHUB_OUTPUT | |
| echo "KART_VERSION=$VER" >> $GITHUB_ENV | |
| echo "is_release=$IS_RELEASE" >> $GITHUB_OUTPUT | |
| - name: "app: configuration & vendor dependencies" | |
| uses: lukka/run-cmake@v10 | |
| env: | |
| LD_LIBRARY_PATH: "${{ env.LD_LIBRARY_PATH }}:${{ env.GITHUB_WORKSPACE }}/build/vcpkg_installed/${{ env.ARCH_TRIPLET }}/lib" | |
| with: | |
| configurePreset: ci-linux | |
| configurePresetAdditionalArgs: ${{ env.OPTIONS_FOR_CMAKE }} | |
| - name: "check for vcpkg errors" | |
| if: "failure() || cancelled()" | |
| shell: bash | |
| run: | | |
| echo "RUNVCPKG_NO_CACHE=1" >> $GITHUB_ENV | |
| - name: "app: build" | |
| shell: bash | |
| run: | | |
| cmake --build --preset=ci-linux --verbose --parallel | |
| - name: "vendor: save archive" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: vendor-linux-${{ env.ARCH }}-py${{ env.PY_VER }} | |
| path: build/vcpkg-vendor/kart-vendor.*.tar.gz | |
| if-no-files-found: error | |
| # | |
| # App tests & checks | |
| # | |
| - name: "test: smoke test" | |
| shell: bash | |
| run: | | |
| ./build/kart --version | |
| - name: "test: create no_postgis environment for tests" | |
| shell: bash | |
| run: | | |
| yum install -y postgresql | |
| psql postgresql://postgres:@postgis:5432/postgres -c 'CREATE DATABASE no_postgis;' | |
| echo "KART_NO_POSTGIS_URL=postgresql://postgres:@postgis:5432/no_postgis" >> $GITHUB_ENV | |
| - name: "test: install MSSQL database drivers for tests" | |
| if: matrix.os.arch != 'arm64' | |
| shell: bash | |
| run: | | |
| curl https://packages.microsoft.com/config/rhel/7/prod.repo > /etc/yum.repos.d/mssql-release.repo | |
| ACCEPT_EULA=Y yum install -y msodbcsql18 mssql-tools | |
| echo "KART_SQLSERVER_URL=${SQLSERVER_URL}" >> $GITHUB_ENV | |
| - name: "test: unit tests" | |
| shell: bash | |
| run: | | |
| ctest --preset=ci-linux | |
| # | |
| # Packaging | |
| # | |
| - name: "bundle: assemble" | |
| shell: bash | |
| run: | | |
| cmake --build --preset=ci-bundle-linux --verbose | |
| - name: "bundle: smoke test" | |
| shell: bash | |
| run: | | |
| ./build/pyinstaller/dist/kart/kart --version | |
| - name: "bundle: e2e tests" | |
| shell: bash | |
| run: | | |
| ctest --preset=ci-e2e-linux | |
| - name: "package: TGZ & DEB & RPM" | |
| shell: bash | |
| run: | | |
| cpack --preset=ci-linux | |
| - name: "package: save TGZ" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Kart-linux-${{ matrix.os.arch }}-tgz | |
| path: build/dist/Kart-*.tar.gz | |
| if-no-files-found: error | |
| - name: "package: save DEB" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Kart-linux-${{ matrix.os.arch }}-deb | |
| path: build/dist/kart_*.deb | |
| if-no-files-found: error | |
| - name: "package: save RPM" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Kart-linux-${{ matrix.os.arch }}-rpm | |
| path: build/dist/kart-*.rpm | |
| if-no-files-found: error | |
| - name: "release" | |
| uses: softprops/action-gh-release@v1 | |
| if: "steps.version.outputs.is_release == 1" | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| draft: true | |
| files: | | |
| build/dist/Kart-*.tar.gz | |
| build/dist/kart_*.deb | |
| build/dist/kart-*.rpm | |
| # | |
| # Linux Package tests | |
| # Install on a wide variety of Linux distributions and run E2E tests | |
| # | |
| Linux-Package-Tests: | |
| needs: ["Linux-Package"] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arch: | |
| - label: amd64 | |
| runner: ubuntu-22.04 | |
| - label: arm64 | |
| runner: buildjet-2vcpu-ubuntu-2204-arm | |
| os: | |
| - label: "tgz-ubuntu-22.04" | |
| image: "ubuntu:jammy" | |
| type: "tgz" | |
| - label: "ubuntu-22.04" | |
| image: "ubuntu:jammy" | |
| type: "deb" | |
| - label: "ubuntu-20.04" | |
| image: "ubuntu:focal" | |
| type: "deb" | |
| - label: "ubuntu-18.04" | |
| image: "ubuntu:bionic" | |
| type: "deb" | |
| - label: "debian-stable" | |
| image: "debian:stable-slim" | |
| type: "deb" | |
| - label: "debian-oldstable" | |
| image: "debian:oldstable-slim" | |
| type: "deb" | |
| - label: "amazonlinux-2023" | |
| image: "amazonlinux:2023" | |
| type: "rpm" | |
| - label: "fedora-latest" | |
| image: "fedora:latest" | |
| type: "rpm" | |
| - label: "almalinux-9" | |
| image: "almalinux:9" | |
| type: "rpm" | |
| - label: "almalinux-8" | |
| image: "almalinux:8" | |
| type: "rpm" | |
| runs-on: ${{ matrix.arch.runner }} | |
| name: "Install-check: ${{ matrix.os.label }}/${{ matrix.arch.label }}" | |
| continue-on-error: ${{ !!matrix.os.xfail }} | |
| container: | |
| image: ${{ matrix.os.image }} | |
| env: | |
| DEBIAN_FRONTEND: noninteractive | |
| steps: | |
| - name: setup | |
| shell: bash | |
| run: | | |
| case "${{ matrix.os.type }}" in | |
| deb | tgz) | |
| apt update -q | |
| apt install -q -y --no-install-recommends sqlite3 | |
| ;; | |
| rpm) | |
| yum install -y sqlite tar | |
| ;; | |
| *) | |
| exit 1 | |
| ;; | |
| esac | |
| - name: get kart installer | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: Kart-linux-${{ matrix.arch.label }}-${{ matrix.os.type }} | |
| - name: install kart | |
| shell: bash | |
| run: | | |
| case "${{ matrix.os.type }}" in | |
| deb) | |
| apt install -y ./kart_*.deb | |
| ;; | |
| rpm) | |
| yum install -y ./kart-*.rpm | |
| ;; | |
| tgz) | |
| tar -xz -C /opt --strip-components=1 -f ./Kart-*.tar.gz | |
| ln -sf /opt/kart/kart /usr/local/bin/kart | |
| ;; | |
| *) | |
| exit 1 | |
| ;; | |
| esac | |
| - name: smoke test | |
| shell: bash | |
| run: | | |
| /opt/kart/kart --version | |
| command -v kart | |
| kart --version | |
| - uses: actions/checkout@v3 | |
| - name: e2e test (no helper) | |
| env: | |
| KART_USE_HELPER: "0" | |
| shell: bash | |
| run: | | |
| time tests/scripts/e2e-1.sh | |
| - name: e2e test (helper) | |
| env: | |
| KART_USE_HELPER: "1" | |
| shell: bash | |
| run: | | |
| time tests/scripts/e2e-1.sh | |
| # | |
| # Linux development build using vendor archive & host python | |
| # | |
| Linux-dev-vendor: | |
| name: "Build Linux dev+vendor" | |
| needs: ["Linux-Package"] | |
| runs-on: ubuntu-22.04 | |
| # We want to run on external PRs, but not on our own internal PRs as they'll be run | |
| # by the push to the branch. | |
| # https://github.community/t/duplicate-checks-on-push-and-pull-request-simultaneous-event/18012/7 | |
| if: > | |
| ( | |
| github.event_name == 'push' | |
| || github.event.pull_request.head.repo.full_name != github.repository | |
| ) | |
| env: | |
| CCACHE_DIR: ${{ github.workspace }}/.cache/ccache | |
| CCACHE_COMPRESS: "1" | |
| PY_VER_ID: "cp310-cp310" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| # | |
| # setup | |
| # | |
| - name: "setup: python" | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PY_VER }} | |
| cache: 'pip' | |
| cache-dependency-path: 'requirements/*.txt' | |
| - name: "setup: cmake & ninja" | |
| uses: lukka/get-cmake@arm64-win-linux | |
| with: | |
| cmakeVersion: "${{ env.CMAKE_VERSION }}" | |
| - name: "setup: ccache" | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ env.CCACHE_DIR }} | |
| key: vendor-ccache-ubuntu-22.04-cmake-vcpkg | |
| - name: "setup: misc" | |
| run: | | |
| sudo apt-get update -q -y | |
| sudo apt-get install -q -y --no-install-recommends ccache unixodbc file sqlite3 | |
| mkdir -p ${{ env.CCACHE_DIR }} | |
| echo "/usr/lib/ccache" >> $GITHUB_PATH | |
| # | |
| # App Build | |
| # | |
| - name: "app: version" | |
| id: version | |
| run: | | |
| if [[ '${{ github.ref_type }}' == 'tag' ]] && [[ '${{ github.ref_name }}' =~ ^v(.*) ]]; then | |
| VER=$(echo ${{ github.ref_name }} | sed 's/^v//') | |
| IS_RELEASE=1 | |
| else | |
| VER=$(sed -E "s/(.*)/\1+ci.${{ github.run_number }}.git${GITHUB_SHA::8}/" kart/VERSION) | |
| IS_RELEASE=0 | |
| fi | |
| echo "App Version: $VER" | |
| echo "Is Release? $IS_RELEASE" | |
| echo "value=$VER" >> $GITHUB_OUTPUT | |
| echo "KART_VERSION=$VER" >> $GITHUB_ENV | |
| echo "is_release=$IS_RELEASE" >> $GITHUB_OUTPUT | |
| - name: "app: get vendor archive" | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: vendor-linux-amd64-py${{ env.PY_VER }} | |
| - name: "app: configuration & vendor dependencies" | |
| shell: bash | |
| run: | | |
| cmake -B ./build -S . \ | |
| -DUSE_VCPKG=NO \ | |
| -DVENDOR_ARCHIVE=./kart-vendor.${PY_VER_ID}-linux_x86_64.tar.gz \ | |
| -DBUILD_TESTING=YES | |
| - name: "app: build" | |
| shell: bash | |
| run: | | |
| cmake --build ./build --verbose | |
| # | |
| # App tests & checks | |
| # | |
| - name: "test: smoke test" | |
| shell: bash | |
| run: | | |
| ./build/kart --version | |
| - name: "bundle: assemble" | |
| shell: bash | |
| run: | | |
| cmake --build ./build --target bundle --verbose | |
| - name: "bundle: smoke test" | |
| shell: bash | |
| run: | | |
| ./build/pyinstaller/dist/kart/kart --version | |
| - name: "bundle: e2e tests" | |
| shell: bash | |
| working-directory: ./build | |
| run: | | |
| ctest -V -L e2e | |
| # | |
| # macOS | |
| # | |
| macOS: | |
| strategy: | |
| matrix: | |
| os: [{id: macos-12, label: "12/x86_64"}] | |
| runs-on: ${{ matrix.os.id }} | |
| name: macOS ${{ matrix.os.label }} | |
| # We want to run on external PRs, but not on our own internal PRs as they'll be run | |
| # by the push to the branch. | |
| # https://github.community/t/duplicate-checks-on-push-and-pull-request-simultaneous-event/18012/7 | |
| if: > | |
| ( | |
| github.event_name == 'push' | |
| || github.event.pull_request.head.repo.full_name != github.repository | |
| ) | |
| env: | |
| CCACHE_DIR: ${{ github.workspace }}/.cache/ccache | |
| CCACHE_COMPRESS: "1" | |
| HOMEBREW_CACHE: ${{ github.workspace }}/.cache/brew | |
| HOMEBREW_NO_INSTALL_CLEANUP: "1" | |
| MACOS_CODESIGN_ID: ${{ secrets.MACOS_CODESIGN_ID }} | |
| MACOS_PKGSIGN_ID: ${{ secrets.MACOS_PKGSIGN_ID }} | |
| MACOS_NOTARIZE_KEYCHAIN_PROFILE: "NOTARIZE_AUTH" | |
| # X.Y version needs to match PY_VER: | |
| PY_VER_INSTALLER: "https://www.python.org/ftp/python/3.10.9/python-3.10.9-macos11.pkg" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| # | |
| # setup | |
| # | |
| - uses: maxim-lobanov/setup-xcode@v1 | |
| with: | |
| xcode-version: latest-stable | |
| # TODO: cache | |
| - name: "setup: python" | |
| run: | | |
| curl -sSfL -o ${{runner.temp}}/py.pkg "${PY_VER_INSTALLER}" | |
| sudo installer -pkg ${{runner.temp}}/py.pkg -target / | |
| sudo ln -sf python3 /Library/Frameworks/Python.framework/Versions/${PY_VER}/bin/python | |
| export PATH="/Library/Frameworks/Python.framework/Versions/${PY_VER}/bin:$PATH" | |
| which python3 | |
| python3 -V --version | |
| which python | |
| python -V --version | |
| echo /Library/Frameworks/Python.framework/Versions/${PY_VER}/bin >> $GITHUB_PATH | |
| - name: "setup: homebrew cache" | |
| id: cache-brew | |
| uses: actions/cache@v3 | |
| with: | |
| path: .cache/brew | |
| key: brew-cmake-${{ matrix.os.id }}-vcpkg | |
| - name: "setup: cmake & ninja" | |
| uses: lukka/get-cmake@latest | |
| with: | |
| cmakeVersion: "${{ env.CMAKE_VERSION }}" | |
| - name: "setup: ccache" | |
| uses: actions/cache@v3 | |
| if: steps.cache-vendor-dist.outputs.cache-hit != 'true' | |
| with: | |
| path: ${{ env.CCACHE_DIR }} | |
| key: vendor-ccache-${{ matrix.os.id }}-vcpkg | |
| - name: "setup: misc" | |
| run: | | |
| brew install -q autoconf automake ccache pandoc pkg-config | |
| mkdir -p ${{ env.CCACHE_DIR }} | |
| echo "$(brew --prefix)/opt/ccache/libexec" >> $GITHUB_PATH | |
| - name: "setup: vcpkg" | |
| uses: lukka/run-vcpkg@v10 | |
| with: | |
| vcpkgDirectory: "${{ github.workspace }}/vcpkg-vendor/vcpkg" | |
| vcpkgJsonGlob: "**/vcpkg-vendor/vcpkg.json" | |
| prependedCacheKey: "20221110a" | |
| appendedCacheKey: "C${{ env.CMAKE_VERSION}}:${{ hashFiles('vcpkg-vendor/vcpkg-overlay-triplets/**', 'vcpkg-vendor/vcpkg-overlay-ports/**', 'vcpkg-vendor/vcpkg-toolchain-setup.cmake') }}" | |
| # | |
| # App Build | |
| # | |
| - name: "app: version" | |
| id: version | |
| env: | |
| MACOS_NOTARIZE_TEAMID: ${{ secrets.MACOS_NOTARIZE_TEAMID }} | |
| run: | | |
| if [[ '${{ github.ref_type }}' == 'tag' ]] && [[ '${{ github.ref_name }}' =~ ^v(.*) ]]; then | |
| VER=$(echo ${{ github.ref_name }} | sed 's/^v//') | |
| IS_RELEASE=1 | |
| else | |
| VER=$(sed -E "s/(.*)/\1+ci.${{ github.run_number }}.git${GITHUB_SHA::8}/" kart/VERSION) | |
| IS_RELEASE=0 | |
| fi | |
| echo "App Version: $VER" | |
| echo "Is Release? $IS_RELEASE" | |
| echo "value=$VER" >> $GITHUB_OUTPUT | |
| echo "KART_VERSION=$VER" >> $GITHUB_ENV | |
| echo "is_release=$IS_RELEASE" >> $GITHUB_OUTPUT | |
| if [[ -n "$MACOS_CODESIGN_ID" ]]; then | |
| echo "MACOS_SIGN_BUNDLE=ON" >> $GITHUB_ENV | |
| else | |
| echo "MACOS_SIGN_BUNDLE=OFF" >> $GITHUB_ENV | |
| fi | |
| if (( $IS_RELEASE )) && [[ -n "$MACOS_PKGSIGN_ID" ]]; then | |
| echo "MACOS_SIGN_PKG=ON" >> $GITHUB_ENV | |
| else | |
| echo "MACOS_SIGN_PKG=OFF" >> $GITHUB_ENV | |
| fi | |
| if (( $IS_RELEASE )) && [[ -n "$MACOS_NOTARIZE_TEAMID" ]]; then | |
| echo "MACOS_NOTARIZE=ON" >> $GITHUB_ENV | |
| else | |
| echo "MACOS_NOTARIZE=OFF" >> $GITHUB_ENV | |
| fi | |
| - name: "app: configuration & vendor dependencies" | |
| uses: lukka/run-cmake@v10 | |
| with: | |
| configurePreset: ci-macos | |
| - name: "check for vcpkg errors" | |
| if: failure() || cancelled() | |
| run: | | |
| echo "RUNVCPKG_NO_CACHE=1" >> $GITHUB_ENV | |
| - name: "app: build" | |
| uses: lukka/run-cmake@v10 | |
| with: | |
| configurePreset: ci-macos | |
| buildPreset: ci-macos | |
| - name: "vendor: save archive" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: vendor-macos-${{ runner.arch }}-py${{ env.PY_VER }} | |
| path: build/vcpkg-vendor/kart-vendor.*.tar.gz | |
| if-no-files-found: error | |
| # | |
| # App tests & checks | |
| # | |
| - name: "test: smoke test" | |
| run: | | |
| ./build/kart --version | |
| - name: "test: install clone_checker for reflink testing" | |
| run: | | |
| git clone https://github.com/dyorgio/apfs-clone-checker.git | |
| cd apfs-clone-checker | |
| gcc clone_checker.c -o clone_checker | |
| chmod +x clone_checker | |
| mv clone_checker /usr/local/bin/ | |
| - name: "test: unit tests" | |
| uses: lukka/run-cmake@v10 | |
| with: | |
| configurePreset: ci-macos | |
| testPreset: ci-macos | |
| # | |
| # Packaging | |
| # | |
| - name: "bundle: setup app signing certificate" | |
| id: keychain | |
| uses: apple-actions/import-codesign-certs@v1 | |
| if: env.MACOS_SIGN_BUNDLE == 'ON' | |
| with: | |
| p12-file-base64: ${{ secrets.MACOS_APP_CERT }} | |
| p12-password: ${{ secrets.MACOS_CERT_PW }} | |
| - name: "bundle: setup notarization auth" | |
| if: env.MACOS_NOTARIZE == 'ON' | |
| env: | |
| MACOS_NOTARIZE_TEAMID: ${{ secrets.MACOS_NOTARIZE_TEAMID }} | |
| MACOS_NOTARIZE_USER: ${{ secrets.MACOS_NOTARIZE_USER }} | |
| MACOS_NOTARIZE_PW: ${{ secrets.MACOS_NOTARIZE_PW }} | |
| run: | | |
| xcrun notarytool store-credentials "$MACOS_NOTARIZE_KEYCHAIN_PROFILE" \ | |
| --apple-id "$MACOS_NOTARIZE_USER" \ | |
| --team-id "$MACOS_NOTARIZE_TEAMID" \ | |
| --password "$MACOS_NOTARIZE_PW" | |
| - name: "bundle: assemble" | |
| uses: lukka/run-cmake@v10 | |
| with: | |
| configurePreset: ci-macos | |
| buildPreset: ci-bundle-macos | |
| - name: "bundle: smoke test" | |
| shell: bash | |
| run: | | |
| ./build/pyinstaller/dist/Kart.app/Contents/MacOS/kart --version | |
| - name: "bundle: e2e tests" | |
| uses: lukka/run-cmake@v10 | |
| with: | |
| configurePreset: ci-macos | |
| testPreset: ci-e2e-macos | |
| - name: "package: setup installer signing certificate" | |
| uses: apple-actions/import-codesign-certs@v1 | |
| if: env.MACOS_SIGN_PKG == 'ON' | |
| with: | |
| create-keychain: false | |
| keychain-password: ${{ steps.keychain.outputs.keychain-password }} | |
| p12-file-base64: ${{ secrets.MACOS_INSTALLER_CERT }} | |
| p12-password: ${{ secrets.MACOS_CERT_PW }} | |
| - name: "package: ZIP & PKG" | |
| shell: bash | |
| run: | | |
| cpack --preset=ci-macos | |
| - name: "package: save ZIP" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Kart-macOS-${{ runner.arch }}-bundle | |
| path: build/dist/Kart-*.zip | |
| if-no-files-found: error | |
| - name: "package: save PKG" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Kart-macOS-${{ runner.arch }}-installer | |
| path: build/dist/Kart-*.pkg | |
| if-no-files-found: error | |
| - name: "package: tests" | |
| run: | | |
| sudo installer -pkg build/dist/Kart-*.pkg -dumplog -target / | |
| readlink $(which kart) | |
| # make sure good sqlite is in path | |
| export PATH="$(pwd)/build/vcpkg_installed/x64-osx/tools:$PATH" | |
| tests/scripts/e2e-1.sh | |
| - name: "release" | |
| uses: softprops/action-gh-release@v1 | |
| if: "steps.version.outputs.is_release == 1" | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| draft: true | |
| files: | | |
| build/dist/Kart-*.zip | |
| build/dist/Kart-*.pkg | |
| # | |
| # Windows | |
| # | |
| Windows: | |
| strategy: | |
| matrix: | |
| os: [{id: windows-2019, label: "2019/x64"}] | |
| runs-on: ${{ matrix.os.id }} | |
| name: Windows ${{ matrix.os.label }} | |
| env: | |
| NINJA_VERSION: "~1.10.2" # workaround for python logging output buffering noise | |
| SIGN_AZURE_CERTIFICATE: ${{ secrets.WIN_SIGN_AZURE_CERTIFICATE }} | |
| # We want to run on external PRs, but not on our own internal PRs as they'll be run | |
| # by the push to the branch. | |
| # https://github.community/t/duplicate-checks-on-push-and-pull-request-simultaneous-event/18012/7 | |
| if: > | |
| ( | |
| github.event_name == 'push' | |
| || github.event.pull_request.head.repo.full_name != github.repository | |
| ) | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| # Setup | |
| - name: "msvc setup" | |
| uses: ilammy/msvc-dev-cmd@v1 | |
| - name: "Harden vcvarsall.bat" | |
| shell: pwsh | |
| run: | | |
| cd 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build' | |
| mv vcvarsall.bat orig_vcvarsall.bat | |
| echo @' | |
| @if not "%VSCMD_DEBUG%" GEQ "3" echo off | |
| FOR /F "tokens=*" %%g IN ('powershell -Command "($Env:PATH.split(';') | Select -Unique) -Join ';'"') do (SET PATH=%%g) | |
| '@ > vcvarsall.bat | |
| type orig_vcvarsall.bat >> vcvarsall.bat | |
| - name: "setup: cmake" | |
| uses: lukka/get-cmake@latest | |
| with: | |
| cmakeVersion: "${{ env.CMAKE_VERSION }}" | |
| ninjaVersion: "${{ env.NINJA_VERSION }}" | |
| - name: "setup: python" | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "${{ env.PY_VER }}" | |
| architecture: x64 | |
| cache: 'pip' | |
| cache-dependency-path: 'requirements/*.txt' | |
| - name: "setup: vcpkg" | |
| uses: lukka/run-vcpkg@v10 | |
| with: | |
| vcpkgDirectory: "${{ github.workspace }}/vcpkg-vendor/vcpkg" | |
| vcpkgJsonGlob: "**/vcpkg-vendor/vcpkg.json" | |
| prependedCacheKey: "20221201a" | |
| appendedCacheKey: "C${{ env.CMAKE_VERSION}}:N${{ env.NINJA_VERSION }}:${{ hashFiles('vcpkg-vendor/vcpkg-overlay-triplets/**', 'vcpkg-vendor/vcpkg-overlay-ports/**', 'vcpkg-vendor/vcpkg-toolchain-setup.cmake') }}" | |
| - name: "setup: misc" | |
| shell: pwsh | |
| run: | | |
| & dotnet tool install --global AzureSignTool --version 3.0.0 | |
| & choco install sqlite.shell | |
| # | |
| # App Build | |
| # | |
| - name: "app: version" | |
| id: version | |
| shell: bash | |
| run: | | |
| if [[ '${{ github.ref_type }}' == 'tag' ]] && [[ '${{ github.ref_name }}' =~ ^v(.*) ]]; then | |
| VER=$(echo ${{ github.ref_name }} | sed 's/^v//') | |
| IS_RELEASE=1 | |
| else | |
| VER=$(sed -E "s/(.*)/\1+ci.${{ github.run_number }}.git${GITHUB_SHA::8}/" kart/VERSION) | |
| IS_RELEASE=0 | |
| fi | |
| echo "App Version: $VER" | |
| echo "Is Release? $IS_RELEASE" | |
| echo "value=$VER" >> $GITHUB_OUTPUT | |
| echo "KART_VERSION=$VER" >> $GITHUB_ENV | |
| echo "is_release=$IS_RELEASE" >> $GITHUB_OUTPUT | |
| if (( $IS_RELEASE )) && [[ -n "$SIGN_AZURE_CERTIFICATE" ]]; then | |
| echo "WIN_SIGN_BUNDLE=ON" >> $GITHUB_ENV | |
| echo "WIN_SIGN_INSTALLER=ON" >> $GITHUB_ENV | |
| else | |
| echo "WIN_SIGN_BUNDLE=OFF" >> $GITHUB_ENV | |
| echo "WIN_SIGN_INSTALLER=OFF" >> $GITHUB_ENV | |
| fi | |
| - name: "app: configuration & vendor dependencies" | |
| uses: lukka/run-cmake@v10 | |
| with: | |
| configurePreset: ci-windows | |
| - name: "check for vcpkg errors" | |
| if: failure() || cancelled() | |
| run: | | |
| echo "RUNVCPKG_NO_CACHE=1" >> $GITHUB_ENV | |
| - name: "app: build" | |
| uses: lukka/run-cmake@v10 | |
| with: | |
| configurePreset: ci-windows | |
| buildPreset: ci-windows | |
| - name: "vendor: save archive" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: vendor-windows-${{ runner.arch }}-py${{ env.PY_VER }} | |
| path: build/vcpkg-vendor/kart-vendor.*.zip | |
| if-no-files-found: error | |
| # | |
| # App tests & checks | |
| # | |
| - name: "test: smoke test" | |
| shell: bash | |
| run: | | |
| ./build/kart.cmd --version | |
| - name: "test: unit tests" | |
| uses: lukka/run-cmake@v10 | |
| with: | |
| configurePreset: ci-windows | |
| testPreset: ci-windows | |
| # | |
| # Packaging | |
| # | |
| - name: "bundle: assemble" | |
| uses: lukka/run-cmake@v10 | |
| env: | |
| SIGN_AZURE_VAULT: ${{ secrets.WIN_SIGN_AZURE_VAULT }} | |
| SIGN_AZURE_CLIENTID: ${{ secrets.WIN_SIGN_AZURE_CLIENTID }} | |
| SIGN_AZURE_CLIENTSECRET: ${{ secrets.WIN_SIGN_AZURE_CLIENTSECRET }} | |
| SIGN_AZURE_TENANTID: ${{ secrets.WIN_SIGN_AZURE_TENANTID }} | |
| with: | |
| configurePreset: ci-windows | |
| buildPreset: ci-bundle-windows | |
| - name: "bundle: smoke test" | |
| shell: bash | |
| run: | | |
| ./build/pyinstaller/dist/kart/kart.exe --version | |
| - name: "bundle: e2e tests" | |
| uses: lukka/run-cmake@v10 | |
| with: | |
| configurePreset: ci-windows | |
| testPreset: ci-e2e-windows | |
| - name: "package: ZIP & MSI" | |
| env: | |
| SIGN_AZURE_VAULT: ${{ secrets.WIN_SIGN_AZURE_VAULT }} | |
| SIGN_AZURE_CLIENTID: ${{ secrets.WIN_SIGN_AZURE_CLIENTID }} | |
| SIGN_AZURE_CLIENTSECRET: ${{ secrets.WIN_SIGN_AZURE_CLIENTSECRET }} | |
| SIGN_AZURE_TENANTID: ${{ secrets.WIN_SIGN_AZURE_TENANTID }} | |
| shell: bash | |
| run: | | |
| if ! cpack --preset=ci-windows; then | |
| cat build/dist/_CPack_Packages/win64/WIX/wix.log | |
| exit 1 | |
| fi | |
| - name: "package: save ZIP" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Kart-windows-${{ runner.arch }}-bundle | |
| path: build/dist/Kart-*.zip | |
| if-no-files-found: error | |
| - name: "package: save MSI" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Kart-windows-${{ runner.arch }}-installer | |
| path: build/dist/Kart-*.msi | |
| if-no-files-found: error | |
| - name: "package: tests" | |
| shell: pwsh | |
| run: | | |
| $ErrorView = 'NormalView' | |
| $msi = Resolve-Path -Path ".\build\dist\Kart-*.msi" | Select-Object -ExpandProperty Path | |
| $sqlitePath = Resolve-Path -Path ".\build\vcpkg_installed\x64-windows\tools" | Select-Object -ExpandProperty Path | |
| Write-Host "Installing $msi ..." -ForegroundColor Green | |
| & "msiexec.exe" /I $msi /quiet /norestart /l* install.log | Out-Default | |
| if (!$?) { | |
| Get-Content install.log | |
| throw "MSI install failed" | |
| } | |
| Write-Host "Refreshing system path..." -ForegroundColor Green | |
| $env:Path = [System.Environment]::GetEnvironmentVariable("Path", "Machine") + ";" +[System.Environment]::GetEnvironmentVariable("Path", "User") + ";" + $sqlitePath | |
| Write-Host "Checking Kart runs..." -ForegroundColor Green | |
| & "kart" --version | Out-Default | |
| if (!$?) {throw "Smoke test Failed"} | |
| Write-Host "Running Kart E2E tests..." -ForegroundColor Green | |
| & "tests\scripts\e2e-1.ps1" | Out-Default | |
| if (!$?) {throw "E2E test Failed"} | |
| - name: "release" | |
| uses: softprops/action-gh-release@v1 | |
| if: "steps.version.outputs.is_release == 1" | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| draft: true | |
| files: | | |
| build/dist/Kart-*.zip | |
| build/dist/Kart-*.msi |