From 6999276aa80afed780718a9ef81dfe0ea0f84b8a Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 5 Sep 2024 08:27:23 +0200 Subject: [PATCH 01/13] BLD: assorted cleanups in release workflow --- .github/workflows/release.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 56324590..5fde8ac1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -263,9 +263,6 @@ jobs: artifact: pyogrio-wheel-arm64-osx-dynamic-release steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Set up Python uses: actions/setup-python@v5 with: @@ -313,7 +310,6 @@ jobs: shell: bash # virtual environment is automatically activated run: | - cd .. uv run python -c "import pyogrio; print(f'GDAL version: {pyogrio.__gdal_version__}\nGEOS version: {pyogrio.__gdal_geos_version__}')" uv run python -m pytest --pyargs pyogrio.tests -v From cb3bf8546054a1d86500311d2ecfab3b8a949fc7 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 5 Sep 2024 08:33:13 +0200 Subject: [PATCH 02/13] add scope to cache --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5fde8ac1..39b7c631 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -131,8 +131,8 @@ jobs: tags: manylinux-vcpkg-gdal:latest push: false load: true - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=${{ matrix.container }} + cache-to: type=gha,mode=max,scope=${{ matrix.container }} env: BUILDKIT_PROGRESS: plain From 422488b16ca339ecc28536ddd0dade96b688b04c Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 5 Sep 2024 09:14:02 +0200 Subject: [PATCH 03/13] still checkout the ci files --- .github/workflows/release.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 39b7c631..7c74a385 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -263,6 +263,11 @@ jobs: artifact: pyogrio-wheel-arm64-osx-dynamic-release steps: + - name: Checkout + uses: actions/checkout@v4 + with: + sparse-checkout: ci + - name: Set up Python uses: actions/setup-python@v5 with: From 4d5736db427acfe0406ba0233c02da96f3f95b68 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 5 Sep 2024 09:25:26 +0200 Subject: [PATCH 04/13] cleanup requirements file --- ci/requirements-wheel-test.txt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ci/requirements-wheel-test.txt b/ci/requirements-wheel-test.txt index 35704e3a..b9000e60 100644 --- a/ci/requirements-wheel-test.txt +++ b/ci/requirements-wheel-test.txt @@ -1,9 +1,8 @@ -# pytest 8.0 gives permission error (https://github.com/pytest-dev/pytest/issues/11904) -pytest<8 +pytest # dependencies of geopandas (installed separately with --no-deps to avoid fiona) pandas -pyproj ; (python_version < '3.12') or (python_full_version >= '3.12.1') -shapely>=2 ; (python_version < '3.12') or (python_full_version >= '3.12.1') +pyproj +shapely>=2 packaging # optional test dependencies -pyarrow ; (python_version < '3.12') or (python_full_version >= '3.12.1') +pyarrow From fda02058a4b984e1e310227783d47ceed0daac89 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 5 Sep 2024 10:52:03 +0200 Subject: [PATCH 05/13] correct sparse checkout, debug vcpgk --- .github/workflows/release.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7c74a385..04ea30e8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -209,7 +209,7 @@ jobs: VCPKG_DEFAULT_TRIPLET: ${{ matrix.triplet }} shell: bash run: | - vcpkg install --overlay-triplets=./ci/custom-triplets --feature-flags="versions,manifests" --x-manifest-root=./ci --x-install-root=$VCPKG_INSTALLATION_ROOT/installed + vcpkg install --overlay-triplets=./ci/custom-triplets --feature-flags="versions,manifests" --x-manifest-root=./ci --x-install-root=$VCPKG_INSTALLATION_ROOT/installed --debug vcpkg list - name: Upload vcpkg build logs @@ -266,7 +266,8 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - sparse-checkout: ci + sparse-checkout: ci/requirements-wheel-test.txt + sparse-checkout-cone-mode: false - name: Set up Python uses: actions/setup-python@v5 From 5b95018b543bbd7c7cb4bec9ae410ed4a211e829 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 5 Sep 2024 11:50:41 +0200 Subject: [PATCH 06/13] bump cache --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 04ea30e8..719a6515 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -184,7 +184,7 @@ jobs: path: | ${{ matrix.vcpkg_cache }} # bump the last digit to avoid using previous build cache - key: ${{ matrix.os }}-${{ matrix.arch }}-vcpkg-gdal3.9.2-cache0 + key: ${{ matrix.os }}-${{ matrix.arch }}-vcpkg-gdal3.9.2-cache1 # MacOS build requires aclocal, which is part of automake, but appears # to be missing in default image From e66d1f02fa51aa71009dfaf6781e27ab270d593d Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 5 Sep 2024 13:32:24 +0200 Subject: [PATCH 07/13] try github actions binary cache for vcpkg --- .github/workflows/release.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 719a6515..455e38df 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -90,7 +90,6 @@ jobs: shell: bash # virtual environment is automatically activated run: | - cd .. uv run python -c "import pyogrio; print(f'GDAL version: {pyogrio.__gdal_version__}\nGEOS version: {pyogrio.__gdal_geos_version__}')" uv run python -m pytest --pyargs pyogrio.tests -v @@ -177,14 +176,12 @@ jobs: with: fetch-depth: 0 - - name: Cache vcpkg - uses: actions/cache@v4 - id: vcpkgcache + - name: Export GitHub Actions cache environment variables + uses: actions/github-script@v7 with: - path: | - ${{ matrix.vcpkg_cache }} - # bump the last digit to avoid using previous build cache - key: ${{ matrix.os }}-${{ matrix.arch }}-vcpkg-gdal3.9.2-cache1 + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); # MacOS build requires aclocal, which is part of automake, but appears # to be missing in default image @@ -207,6 +204,7 @@ jobs: - name: Install GDAL env: VCPKG_DEFAULT_TRIPLET: ${{ matrix.triplet }} + VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite" shell: bash run: | vcpkg install --overlay-triplets=./ci/custom-triplets --feature-flags="versions,manifests" --x-manifest-root=./ci --x-install-root=$VCPKG_INSTALLATION_ROOT/installed --debug From 6f3d98110138d8a15fb9eca3689f0483c5467f0a Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 5 Sep 2024 14:48:47 +0200 Subject: [PATCH 08/13] trigger rebuild to test cache --- .github/workflows/release.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 455e38df..154e75d7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -283,8 +283,6 @@ jobs: curl -LsSf https://astral.sh/uv/install.sh | sh . $HOME/.cargo/env uv venv .venv - echo "VIRTUAL_ENV=.venv" >> $GITHUB_ENV - echo "$PWD/.venv/bin" >> $GITHUB_PATH - name: Create virtual environment (Windows) if: ${{ runner.os == 'Windows' }} From 5467c2fea7bd419984830d2f565daf470027931b Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 5 Sep 2024 16:24:16 +0200 Subject: [PATCH 09/13] try building arm64 wheels on macos-14 --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 154e75d7..da8a1c5c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -157,7 +157,7 @@ jobs: vcpkg_cache: "/Users/runner/.cache/vcpkg/archives" vcpkg_logs: "/usr/local/share/vcpkg/buildtrees/**/*.log" - - os: "macos-12" + - os: "macos-14" triplet: "arm64-osx-dynamic-release" arch: arm64 vcpkg_cache: "/Users/runner/.cache/vcpkg/archives" From bdabd231271831563114a447f41af7eb084cd951 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 5 Sep 2024 21:24:41 +0200 Subject: [PATCH 10/13] try with run-vcpkg action --- .github/workflows/release.yml | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index da8a1c5c..3bada0f5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -176,13 +176,6 @@ jobs: with: fetch-depth: 0 - - name: Export GitHub Actions cache environment variables - uses: actions/github-script@v7 - with: - script: | - core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); - core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); - # MacOS build requires aclocal, which is part of automake, but appears # to be missing in default image - name: Reinstall automake @@ -191,20 +184,16 @@ jobs: brew reinstall automake echo $(which aclocal) - - name: Checkout specific version of vcpkg - shell: bash - run: | - cd $VCPKG_INSTALLATION_ROOT - # on mac the clone is not clean, otherwise git pull fails - git reset --hard - # pull specific commit with desired GDAL version - git pull - git checkout 73794ce5f63fd138fab999a22959ca7c6305d93c + - name: Install vcpkg and checkout specific version + uses: lukka/run-vcpkg@v11 + with: + vcpkgDirectory: '${{ github.workspace }}/vcpkg' + vcpkgGitCommitId: 73794ce5f63fd138fab999a22959ca7c6305d93c - name: Install GDAL env: VCPKG_DEFAULT_TRIPLET: ${{ matrix.triplet }} - VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite" + VCPKG_INSTALLATION_ROOT: ${{ github.workspace }}/vcpkg shell: bash run: | vcpkg install --overlay-triplets=./ci/custom-triplets --feature-flags="versions,manifests" --x-manifest-root=./ci --x-install-root=$VCPKG_INSTALLATION_ROOT/installed --debug @@ -222,6 +211,7 @@ jobs: env: # CIBW needs to know triplet for the correct install path VCPKG_DEFAULT_TRIPLET: ${{ matrix.triplet }} + VCPKG_INSTALLATION_ROOT: ${{ github.workspace }}/vcpkg CIBW_ARCHS: ${{ matrix.arch }} - uses: actions/upload-artifact@v4 From d31a00630cc0e6770190aaa02589721628116462 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 5 Sep 2024 21:46:07 +0200 Subject: [PATCH 11/13] try fixes --- ci/custom-triplets/arm64-osx-dynamic-release.cmake | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/custom-triplets/arm64-osx-dynamic-release.cmake b/ci/custom-triplets/arm64-osx-dynamic-release.cmake index def957a4..b12affe6 100644 --- a/ci/custom-triplets/arm64-osx-dynamic-release.cmake +++ b/ci/custom-triplets/arm64-osx-dynamic-release.cmake @@ -3,7 +3,7 @@ set(VCPKG_CRT_LINKAGE dynamic) set(VCPKG_CMAKE_SYSTEM_NAME Darwin) set(VCPKG_OSX_ARCHITECTURES arm64) set(VCPKG_BUILD_TYPE release) -set(VCPKG_OSX_DEPLOYMENT_TARGET "12.0") +set(VCPKG_OSX_DEPLOYMENT_TARGET "14.0") set(VCPKG_LIBRARY_LINKAGE static) if(PORT MATCHES "gdal") diff --git a/pyproject.toml b/pyproject.toml index 53084902..16840e4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ MACOSX_DEPLOYMENT_TARGET = "12.0" [tool.cibuildwheel.windows] before-build = "pip install delvewheel" -repair-wheel-command = "delvewheel repair --add-path C:/vcpkg/installed/x64-windows-dynamic-release/bin -w {dest_dir} {wheel}" +repair-wheel-command = "delvewheel repair --add-path $VCPKG_INSTALLATION_ROOT/installed/x64-windows-dynamic-release/bin -w {dest_dir} {wheel}" [tool.cibuildwheel.windows.environment] VCPKG_INSTALL = "$VCPKG_INSTALLATION_ROOT/installed/x64-windows-dynamic-release" From 440873a76be4e09dce4e251aff656b5aa104467e Mon Sep 17 00:00:00 2001 From: "Brendan C. Ward" Date: Mon, 23 Dec 2024 10:38:53 -0800 Subject: [PATCH 12/13] Try again with MacOS 13 and more windows logging --- .github/workflows/release.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5ae668e0..2f147e19 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -157,7 +157,7 @@ jobs: vcpkg_cache: "/Users/runner/.cache/vcpkg/archives" vcpkg_logs: "/usr/local/share/vcpkg/buildtrees/**/*.log" - - os: "macos-14" + - os: "macos-13" triplet: "arm64-osx-dynamic-release" arch: arm64 vcpkg_cache: "/Users/runner/.cache/vcpkg/archives" @@ -196,7 +196,7 @@ jobs: - name: Install vcpkg and checkout specific version uses: lukka/run-vcpkg@v11 with: - vcpkgDirectory: '${{ github.workspace }}/vcpkg' + vcpkgDirectory: "${{ github.workspace }}/vcpkg" vcpkgGitCommitId: 0857a4b08c14030bbe41e80accb2b1fddb047a74 - name: Install GDAL @@ -208,6 +208,14 @@ jobs: vcpkg install --overlay-triplets=./ci/custom-triplets --feature-flags="versions,manifests" --x-manifest-root=./ci --x-install-root=$VCPKG_INSTALLATION_ROOT/installed --debug vcpkg list + # FIXME: remove + - name: Log windows install directory + if: ${{ runner.os == 'Windows' }} + run: | + ls $VCPKG_INSTALLATION_ROOT/installed + ls $VCPKG_INSTALLATION_ROOT/installed/x64-windows-dynamic-release + ls $VCPKG_INSTALLATION_ROOT/installed/x64-windows-dynamic-release/bin + - name: Upload vcpkg build logs if: ${{ failure() }} uses: actions/upload-artifact@v4 From 8e49fec7825a2567670ad9f6e3d90f3f10263f5d Mon Sep 17 00:00:00 2001 From: "Brendan C. Ward" Date: Mon, 23 Dec 2024 11:22:14 -0800 Subject: [PATCH 13/13] Remove non-working windows logging, revert MacOS deployment target version --- .github/workflows/release.yml | 8 -------- ci/custom-triplets/arm64-osx-dynamic-release.cmake | 3 ++- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2f147e19..bc201bb4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -208,14 +208,6 @@ jobs: vcpkg install --overlay-triplets=./ci/custom-triplets --feature-flags="versions,manifests" --x-manifest-root=./ci --x-install-root=$VCPKG_INSTALLATION_ROOT/installed --debug vcpkg list - # FIXME: remove - - name: Log windows install directory - if: ${{ runner.os == 'Windows' }} - run: | - ls $VCPKG_INSTALLATION_ROOT/installed - ls $VCPKG_INSTALLATION_ROOT/installed/x64-windows-dynamic-release - ls $VCPKG_INSTALLATION_ROOT/installed/x64-windows-dynamic-release/bin - - name: Upload vcpkg build logs if: ${{ failure() }} uses: actions/upload-artifact@v4 diff --git a/ci/custom-triplets/arm64-osx-dynamic-release.cmake b/ci/custom-triplets/arm64-osx-dynamic-release.cmake index b12affe6..35d3a2e3 100644 --- a/ci/custom-triplets/arm64-osx-dynamic-release.cmake +++ b/ci/custom-triplets/arm64-osx-dynamic-release.cmake @@ -3,9 +3,10 @@ set(VCPKG_CRT_LINKAGE dynamic) set(VCPKG_CMAKE_SYSTEM_NAME Darwin) set(VCPKG_OSX_ARCHITECTURES arm64) set(VCPKG_BUILD_TYPE release) -set(VCPKG_OSX_DEPLOYMENT_TARGET "14.0") +set(VCPKG_OSX_DEPLOYMENT_TARGET "12.0") set(VCPKG_LIBRARY_LINKAGE static) + if(PORT MATCHES "gdal") set(VCPKG_LIBRARY_LINKAGE dynamic) endif()