Python Wheels #242
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python Wheels | |
| on: | |
| workflow_dispatch: | |
| release: | |
| types: ['released', 'prereleased'] | |
| env: | |
| PACKAGE_VERSION: '1.0.0a20.dev0' | |
| PACKAGE_NAME: alpaqa | |
| jobs: | |
| # First we build the wheels natively (build system == host system). | |
| # This allows us to import the compiled modules, and automatically generate | |
| # stub files for them. Those stub files are then included in the sdist | |
| # (source distribution), to be later included in the cross-compiled packages | |
| # as well (because we can't generate stubs while cross-compiling). | |
| # By building the native wheels first, we can already start testing while the | |
| # cross-compiled versions are being built. | |
| build-sdist: | |
| name: Build sdist | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| matrix: | |
| python-version: ['3.13'] | |
| env: | |
| CCACHE_DIR: ${{ github.workspace }}/.ccache | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| # Ccache | |
| - name: Install ccache | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y ccache | |
| mkdir -p "${{ env.CCACHE_DIR }}" | |
| - name: Cache ccache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CCACHE_DIR }} | |
| key: ${{ runner.os }}-wheel-native-ccache-${{ github.run_id }} | |
| restore-keys: ${{ runner.os }}-wheel-native-ccache | |
| - name: Install Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Python dependencies | |
| run: > | |
| pip install -U | |
| pip build conan ninja 'py-build-cmake~=0.3.1' 'pybind11-stubgen~=2.5.1' 'numpy<3' | |
| # Cache the Conan cache folder to speed up installation of the dependencies. | |
| - name: Cache Conan dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.conan2/p | |
| # Key is unique, to force updating the cache, but we still want the | |
| # cache to be restored, so we use restore-keys with a matching prefix. | |
| key: ${{ runner.os }}-build-sdist-${{ github.sha }} | |
| restore-keys: ${{ runner.os }}-build-sdist- | |
| - name: Prepare Conan configuration | |
| run: | | |
| conan profile detect -f | |
| cat <<- EOF > conan.profile | |
| include(default) | |
| include(${{ github.workspace }}/scripts/ci/alpaqa-python.profile) | |
| [conf] | |
| tools.cmake.cmaketoolchain:generator=Ninja Multi-Config | |
| tools.build:skip_test=true | |
| [buildenv] | |
| CFLAGS=-fdiagnostics-color | |
| CXXFLAGS=-fdiagnostics-color | |
| LDFLAGS=-static-libgcc -static-libstdc++ -flto=auto | |
| EOF | |
| - name: Install Conan recipes | |
| run: | | |
| recipes="${{ github.workspace }}/tttapa-conan-recipes" | |
| git clone https://github.com/tttapa/conan-recipes "$recipes" | |
| conan remote add tttapa-conan-recipes "$recipes" --force | |
| - name: Install Conan dependencies | |
| run: | | |
| for c in Debug Release; do | |
| conan install . --build=missing -pr:h conan.profile -s build_type=$c | |
| done | |
| - name: Build Wheel package | |
| run: python3 -m build -w -C local="scripts/ci/py-build-cmake.toml" | |
| env: | |
| CMAKE_C_COMPILER_LAUNCHER: ccache | |
| CMAKE_CXX_COMPILER_LAUNCHER: ccache | |
| - name: Upload Wheel | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: native-wheels | |
| path: dist/*.whl | |
| retention-days: 1 | |
| - name: Install stubs | |
| run: | | |
| # We install the Python modules and stubs in the source directory | |
| for i in 10 20; do | |
| py-build-cmake --local="scripts/ci/py-build-cmake.toml" \ | |
| configure --index $i | |
| py-build-cmake --local="scripts/ci/py-build-cmake.toml" \ | |
| install --index $i --component python_modules -- --prefix python | |
| py-build-cmake --local="scripts/ci/py-build-cmake.toml" \ | |
| install --index $i --component python_stubs -- --prefix python | |
| done | |
| # Then we remove the binary Python modules (sdist is source only) | |
| while IFS= read -r f || [ -n "$f" ]; do rm -f "$f" | |
| done < build/python-debug/install_manifest_python_modules.txt | |
| while IFS= read -r f || [ -n "$f" ]; do rm -f "$f" | |
| done < build/python-release/install_manifest_python_modules.txt | |
| env: | |
| CMAKE_C_COMPILER_LAUNCHER: ccache | |
| CMAKE_CXX_COMPILER_LAUNCHER: ccache | |
| - name: Create sdist | |
| run: python3 -m build -s | |
| - name: Upload sdist | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: sdist | |
| path: dist/*.tar.gz | |
| retention-days: 1 | |
| # Testing is done in the official Python Docker container: https://hub.docker.com/_/python/ | |
| # This should match more closely to the environment that users might use. | |
| # It also ensures that we don't accidentally depend on any libraries specific | |
| # to the build container. | |
| test-linux: | |
| name: Run tests | |
| needs: [build-sdist] | |
| runs-on: ubuntu-latest | |
| container: python:3.13-bookworm | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Download wheels | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: native-wheels | |
| path: dist | |
| - name: Install | |
| run: python3 -m pip install --find-links=dist "${PACKAGE_NAME}[test]==${PACKAGE_VERSION}" | |
| - name: Test | |
| run: pytest -rP | |
| # After the native build, we have the stub files, and we can start cross- | |
| # compiling for other architectures. | |
| cross-build-linux: | |
| name: Cross-build wheels for ${{ matrix.host }} - ${{ matrix.python-version }} | |
| needs: [build-sdist] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| host: [x86_64-bionic-linux-gnu, aarch64-rpi3-linux-gnu] | |
| python-version: | |
| - python3.13 | |
| - pypy3.10-v7.3 | |
| env: | |
| CCACHE_DIR: ${{ github.workspace }}/.ccache | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| # Ccache | |
| - name: Install ccache | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y ccache | |
| mkdir -p "${{ env.CCACHE_DIR }}" | |
| - name: Cache ccache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CCACHE_DIR }} | |
| key: ${{ runner.os }}-wheel-${{ matrix.host }}-${{ matrix.python-version }}-ccache-${{ github.run_id }} | |
| restore-keys: ${{ runner.os }}-wheel-${{ matrix.host }}-${{ matrix.python-version }}-ccache | |
| - name: Download sdist | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: sdist | |
| path: dist | |
| - name: Extract sdist | |
| run: mkdir sdist && tar xf dist/*.tar.gz -C sdist --strip-components 1 | |
| - name: Build | |
| uses: ./.github/workflows/python-build | |
| with: | |
| source-dir: sdist | |
| host: ${{ matrix.host }} | |
| python-version: ${{ matrix.python-version }} | |
| ccache: ccache | |
| - name: Upload package | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheels-${{ matrix.host }}-${{ matrix.python-version }} | |
| path: ./sdist/dist/*.whl | |
| # Build for Windows and macOS using cibuildwheel. | |
| # Since we're not specifying any cross-compilation settings, py-build-cmake | |
| # will use its default cross-compilation settings for Windows on ARM64. | |
| # For macOS, we build universal wheels that work on both Intel and ARM macs. | |
| build-macos-windows: | |
| name: Build wheels for ${{ matrix.os }} | |
| needs: [build-sdist] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [macos-latest, windows-latest] | |
| fail-fast: false | |
| env: | |
| CONAN_HOME: ${{ github.workspace }}/.conan2 | |
| SCCACHE_GHA_VERSION: "1" | |
| SCCACHE_CACHE_MULTIARCH: "1" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install sccache | |
| uses: mozilla-actions/sccache-action@9e326ebed976843c9932b3aa0e021c6f50310eb4 | |
| - name: Cache Conan dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CONAN_HOME }}/p | |
| key: ${{ runner.os }}-build-wheel-${{ github.sha }} | |
| restore-keys: ${{ runner.os }}-build-wheel- | |
| - name: Download sdist | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: sdist | |
| path: dist | |
| - name: Extract sdist | |
| shell: bash | |
| run: | | |
| mkdir sdist | |
| tar xf dist/*.tar.gz -C sdist --strip-components 1 | |
| cp -a scripts sdist | |
| - name: Build wheels | |
| uses: pypa/cibuildwheel@ee63bf16da6cddfb925f542f2c7b59ad50e93969 | |
| with: | |
| package-dir: sdist | |
| output-dir: dist | |
| env: | |
| CIBW_ENABLE: 'pypy' | |
| CIBW_TEST_SKIP: 'pp*' | |
| CIBW_BUILD: 'cp311-* pp310-*' | |
| - name: Upload package | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheels-${{ matrix.os }} | |
| path: ./dist/*.whl | |
| # This step checks the package version before release (to make sure that the | |
| # package version actually matches the version of the GitHub release tag), | |
| # and uses Twine to check the metadata of the packages. | |
| check-release: | |
| if: ${{ github.event.action == 'released' || github.event.action == 'prereleased' }} | |
| needs: [build-sdist, test-linux, build-macos-windows] | |
| runs-on: ubuntu-latest | |
| container: python:3.12-bullseye | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: wheels-* | |
| path: dist | |
| merge-multiple: true | |
| - name: Install package | |
| run: python -m pip install --no-deps --no-index --find-links=dist ${PACKAGE_NAME}==${PACKAGE_VERSION} | |
| - name: Check package version | |
| run: | | |
| [ "${{ github.event.release.tag_name }}" == $(python -c 'from importlib.metadata import version as v; print(v("${{ env.PACKAGE_NAME }}"))') ] | |
| shell: bash | |
| - name: Twine check | |
| run: | | |
| python -m pip install twine | |
| twine check dist/* | |
| # Here we download the sdist and the built Wheel files, and upload them to | |
| # TestPyPI. You should follow the trusted publishing instructions in the | |
| # https://github.com/pypa/gh-action-pypi-publish README and on | |
| # https://docs.pypi.org/trusted-publishers carefully! | |
| release: | |
| needs: [check-release] | |
| if: ${{ github.event.action == 'released' || github.event.action == 'prereleased' }} | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: testpypi | |
| url: https://pypi.org/p/alpaqa | |
| permissions: | |
| id-token: write # mandatory for trusted publishing | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: wheels-* | |
| path: dist | |
| merge-multiple: true | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: sdist | |
| path: dist | |
| - name: Publish package distributions to PyPI | |
| uses: pypa/gh-action-pypi-publish@15c56dba361d8335944d31a2ecd17d700fc7bcbc |