diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fa8382f..4694c77 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,230 +1,28 @@ name: Build on: - workflow_dispatch: - push: - branches: - - develop/* pull_request: branches: - master release: types: [ published ] -env: - PYTHON_VERSIONS: "3.10,3.11,3.12,3.13,3.14" - concurrency: cancel-in-progress: false - # This creates different concurrency groups depending on the trigger - # For PRs: uses the PR number (e.g., "build-pr-123") - # For branch pushes: uses the branch name (e.g., "build-develop/feature-x") - # For releases: uses the release tag plus a unique run ID so each release runs independently - # For manual dispatches: uses a unique run ID so each dispatch runs independently - group: ${{ github.event_name == 'pull_request' && format('build-pr-{0}', github.event.pull_request.number) || github.event_name == 'push' && format('build-{0}', github.ref) || github.event_name == 'release' && format('build-release-{0}-{1}', github.event.release.tag_name, github.run_id) || format('build-dispatch-{0}', github.run_id) }} - -defaults: - run: - shell: bash + # PRs get one group per PR number; releases get one group per tag, so a release + # build never queues behind or cancels an unrelated PR build. + group: ${{ github.event_name == 'pull_request' && format('build-pr-{0}', github.event.pull_request.number) || format('build-release-{0}', github.event.release.tag_name) }} jobs: - setup: - runs-on: ubuntu-latest - outputs: - python-versions: ${{ steps.set-matrix.outputs.python-versions }} - cibw-build: ${{ steps.set-matrix.outputs.cibw-build }} - steps: - - id: set-matrix - run: | - # Convert "3.10,3.11,3.12" to ["3.10","3.11","3.12"] - python_array=$(echo '["${{ env.PYTHON_VERSIONS }}"]' | sed 's/,/","/g') - echo "python-versions=$python_array" >> $GITHUB_OUTPUT - - # Convert "3.10,3.11,3.12" to "cp310-* cp311-* cp312-*" - cibw=$(echo "${{ env.PYTHON_VERSIONS }}" | sed 's/\.//g; s/,/ /g; s/\([0-9]*\)/cp\1-*/g') - echo "cibw-build=$cibw" >> $GITHUB_OUTPUT - - build_wheels_windows: - name: Build wheels on Windows - needs: setup - runs-on: windows-latest - strategy: - fail-fast: False - matrix: - python-version: ${{ fromJSON(needs.setup.outputs.python-versions) }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - submodules: true - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Build wheels - run: | - python -m pip install -U pip numpy swig wheel setuptools - python setup.py bdist_wheel -d dist - ls -al ./dist - - - name: Place wheels in artifacts folder - uses: actions/upload-artifact@v4 - with: - name: windows-${{ matrix.python-version }} - path: ./dist/*.whl - - build_wheels_unix: - name: Build wheels on ${{ matrix.os }} - needs: setup - runs-on: ${{ matrix.os }} - strategy: - fail-fast: False - matrix: - os: [ ubuntu-latest, ubuntu-24.04-arm, macos-latest, macos-14 ] - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - submodules: true - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - - name: Build wheels - env: - # only build CPython-3.10+ and skip 32-bit builds - CIBW_BUILD: ${{ needs.setup.outputs.cibw-build }} - CIBW_SKIP: "*-manylinux_i686 *-musllinux*" - # use latest build; manylinux_2_28 ships gcc>=12 (manylinux2014's - # gcc 10.2 is too old for numpy>=2.5 source builds) - CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux_2_28_x86_64 - CIBW_MANYLINUX_AARCH64_IMAGE: quay.io/pypa/manylinux_2_28_aarch64 - CIBW_BEFORE_ALL_MACOS: brew install swig - CIBW_BEFORE_BUILD: pip install numpy swig - run: | - python -m pip install -U pip cibuildwheel - python -m cibuildwheel --output-dir dist - ls -R dist - - - name: Place wheels in artifacts folder - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.os }} - path: ./dist/*.whl - - test-wheels: - name: Test wheels - needs: [ setup, build_wheels_windows, build_wheels_unix ] - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ windows-latest, ubuntu-latest, ubuntu-24.04-arm, macos-latest, macos-14 ] - python-version: ${{ fromJSON(needs.setup.outputs.python-versions) }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - submodules: true - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Retrieve packages - uses: actions/download-artifact@v4 - with: - path: dist - - - name: List items - run: | - ls -alR dist - - - name: Test Package Installation - run: | - python -m pip install --upgrade pip - - # list all files in the dist folder - ls -R dist - - # finds path to the right wheel or source file to install later - os=$(echo ${{ runner.os }} | awk '{print tolower($0)}' | head -c3) - - # version refers to the python version. So 3.8 -> 38 - version=$(echo ${{ matrix.python-version }} | sed 's/\.//g') - - if [[ "${{ matrix.os }}" == "windows-latest" ]]; then - chunk="windows-*" - else - chunk="${{matrix.os}}" - fi - - echo "Runner OS: ${{ runner.os }}" - echo "os : ${os}" - echo "version : ${version}" - echo "chunk : ${chunk}" - - # this finds files like - # nlopt-2.6.2-cp36-cp36m-win_amd64.whl - # nlopt-2.6.2-cp36-cp36m-manylinux10_amd64.whl - # nlopt-2.7.1-cp310-cp310-win_amd64.whl - file=$(find dist/${chunk} -name "nlopt-*${version}*${os}*.whl" -type f); - - echo "Installing file: ${file}" - - pip install ${file} - python extern/nlopt/test/t_python.py - - deploy: - name: Deploy packages - runs-on: ubuntu-latest - needs: test-wheels - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + build: + name: Build & test wheels + uses: ./.github/workflows/wc-build.yml permissions: id-token: write contents: read - - steps: - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Retrieve packages - uses: actions/download-artifact@v4 - with: - path: dist - - - name: Install twine - run: | - pip install twine - pip install -U packaging - - - name: Move files to top level directory - run: | - ls -ltR dist - python - << EOF - from pathlib import Path - import shutil - - d = Path('dist') - for f in d.rglob('*.whl'): - shutil.move(f, d / f.name) - - for f in d.iterdir(): - if f.is_dir(): - shutil.rmtree(f) - - EOF - - - name: Upload packages to testpypi - if: ${{ !startsWith(github.ref, 'refs/tags/') }} - run: python -m twine upload --skip-existing --repository testpypi dist/* --verbose - - - name: Upload packages to pypi - if: startsWith(github.ref, 'refs/tags/') - run: python -m twine upload --skip-existing dist/* --verbose + with: + os: "windows-latest,ubuntu-latest,ubuntu-24.04-arm,macos-latest,macos-14" + python-versions: "3.10,3.11,3.12,3.13,3.14" + # PRs just build/test wheels, no publish; a published release goes to prod pypi. + # A testpypi dry-run publish is available on demand via manual-deploy.yml. + target: ${{ github.event_name == 'release' && 'pypi' || 'none' }} diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml new file mode 100644 index 0000000..e0aefae --- /dev/null +++ b/.github/workflows/manual-deploy.yml @@ -0,0 +1,123 @@ +name: Manual Deploy + +on: + workflow_dispatch: + inputs: + target: + description: "Where to publish" + type: choice + options: + - testpypi + - pypi + - none + default: testpypi + version-suffix: + description: "Optional version suffix, e.g. 'dev123' or 'rc1' -> 2.11.0.dev123. Leave blank to auto-generate one for testpypi." + type: string + default: "" + os_windows-latest: + description: "OS: windows-latest" + type: boolean + default: false + os_ubuntu-latest: + description: "OS: ubuntu-latest" + type: boolean + default: false + os_ubuntu-24.04-arm: + description: "OS: ubuntu-24.04-arm" + type: boolean + default: false + os_macos-latest: + description: "OS: macos-latest" + type: boolean + default: false + os_macos-14: + description: "OS: macos-14" + type: boolean + default: false + py3.10: + description: "Python 3.10" + type: boolean + default: false + py3.11: + description: "Python 3.11" + type: boolean + default: false + py3.12: + description: "Python 3.12" + type: boolean + default: false + py3.13: + description: "Python 3.13" + type: boolean + default: false + py3.14: + description: "Python 3.14" + type: boolean + default: false + +concurrency: + cancel-in-progress: false + group: manual-deploy + +jobs: + package-args: + name: Package dispatch inputs + runs-on: ubuntu-latest + outputs: + os: ${{ steps.pack.outputs.os }} + python-versions: ${{ steps.pack.outputs.python-versions }} + steps: + - id: pack + env: + OS_WINDOWS_LATEST: ${{ inputs['os_windows-latest'] }} + OS_UBUNTU_LATEST: ${{ inputs['os_ubuntu-latest'] }} + OS_UBUNTU_ARM: ${{ inputs['os_ubuntu-24.04-arm'] }} + OS_MACOS_LATEST: ${{ inputs['os_macos-latest'] }} + OS_MACOS_14: ${{ inputs['os_macos-14'] }} + PY_310: ${{ inputs['py3.10'] }} + PY_311: ${{ inputs['py3.11'] }} + PY_312: ${{ inputs['py3.12'] }} + PY_313: ${{ inputs['py3.13'] }} + PY_314: ${{ inputs['py3.14'] }} + run: | + os_list="" + [[ "$OS_WINDOWS_LATEST" == "true" ]] && os_list="$os_list,windows-latest" + [[ "$OS_UBUNTU_LATEST" == "true" ]] && os_list="$os_list,ubuntu-latest" + [[ "$OS_UBUNTU_ARM" == "true" ]] && os_list="$os_list,ubuntu-24.04-arm" + [[ "$OS_MACOS_LATEST" == "true" ]] && os_list="$os_list,macos-latest" + [[ "$OS_MACOS_14" == "true" ]] && os_list="$os_list,macos-14" + os_list="${os_list#,}" + + py_list="" + [[ "$PY_310" == "true" ]] && py_list="$py_list,3.10" + [[ "$PY_311" == "true" ]] && py_list="$py_list,3.11" + [[ "$PY_312" == "true" ]] && py_list="$py_list,3.12" + [[ "$PY_313" == "true" ]] && py_list="$py_list,3.13" + [[ "$PY_314" == "true" ]] && py_list="$py_list,3.14" + py_list="${py_list#,}" + + if [[ -z "$os_list" ]]; then + echo "::error::Select at least one OS checkbox before dispatching." + exit 1 + fi + if [[ -z "$py_list" ]]; then + echo "::error::Select at least one Python version checkbox before dispatching." + exit 1 + fi + + echo "os=$os_list" >> $GITHUB_OUTPUT + echo "python-versions=$py_list" >> $GITHUB_OUTPUT + + deploy: + name: Build, test, and deploy + needs: package-args + uses: ./.github/workflows/wc-build.yml + permissions: + id-token: write + contents: read + with: + os: ${{ needs.package-args.outputs.os }} + python-versions: ${{ needs.package-args.outputs.python-versions }} + target: ${{ inputs.target }} + version-suffix: ${{ inputs['version-suffix'] }} diff --git a/.github/workflows/wc-build.yml b/.github/workflows/wc-build.yml new file mode 100644 index 0000000..c87b165 --- /dev/null +++ b/.github/workflows/wc-build.yml @@ -0,0 +1,325 @@ +name: Build, test and optionally deploy wheels + +on: + workflow_call: + inputs: + os: + description: "Comma-separated OS runner labels to build/test, e.g. 'windows-latest,ubuntu-latest'" + type: string + required: true + python-versions: + description: "Comma-separated Python versions to build/test, e.g. '3.10,3.11'" + type: string + required: true + target: + description: "Where to publish: pypi, testpypi, or none" + type: string + default: none + version-suffix: + description: "Optional version suffix, e.g. 'dev123' or 'rc1' -> 2.11.0.dev123. If blank and target=testpypi, one is auto-generated." + type: string + default: "" + +defaults: + run: + shell: bash + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + python-versions: ${{ steps.set-matrix.outputs.python-versions }} + cibw-build: ${{ steps.set-matrix.outputs.cibw-build }} + os-list-all: ${{ steps.set-matrix.outputs.os-list-all }} + os-list-unix: ${{ steps.set-matrix.outputs.os-list-unix }} + build-windows: ${{ steps.set-matrix.outputs.build-windows }} + build-unix: ${{ steps.set-matrix.outputs.build-unix }} + version-suffix: ${{ steps.set-matrix.outputs.version-suffix }} + steps: + - name: Validate target + run: | + case "${{ inputs.target }}" in + pypi|testpypi|none) ;; + *) + echo "::error::Invalid target '${{ inputs.target }}'. Must be one of: pypi, testpypi, none." + exit 1 + ;; + esac + + # A real pypi release can never be re-published under the same version, so fail + # fast here (before spending 20+ minutes building every wheel) if that version is + # already live, rather than discovering it at the upload step. + - name: Checkout repository + if: inputs.target == 'pypi' + uses: actions/checkout@v4 + with: + submodules: true + + - name: Ensure version is not already published on PyPI + if: inputs.target == 'pypi' + env: + VERSION_SUFFIX_INPUT: ${{ inputs['version-suffix'] }} + run: | + major=$(grep -oP 'NLOPT_MAJOR_VERSION\s+"\K[^"]+' extern/nlopt/CMakeLists.txt) + minor=$(grep -oP 'NLOPT_MINOR_VERSION\s+"\K[^"]+' extern/nlopt/CMakeLists.txt) + bugfix=$(grep -oP 'NLOPT_BUGFIX_VERSION\s+"\K[^"]+' extern/nlopt/CMakeLists.txt) + version="$major.$minor.$bugfix" + if [[ -n "$VERSION_SUFFIX_INPUT" ]]; then + version="$version.$VERSION_SUFFIX_INPUT" + fi + echo "Resolved package version: $version" + + status=$(curl -s -o /dev/null -w '%{http_code}' "https://pypi.org/pypi/nlopt/$version/json") + if [[ "$status" == "200" ]]; then + echo "::error::nlopt $version is already published on PyPI. Bump NLOPT_MAJOR_VERSION/NLOPT_MINOR_VERSION/NLOPT_BUGFIX_VERSION in extern/nlopt/CMakeLists.txt, or pass a distinct version-suffix, before publishing again." + exit 1 + fi + + - id: set-matrix + env: + OS_LIST_INPUT: ${{ inputs.os }} + PYTHON_VERSIONS_INPUT: ${{ inputs.python-versions }} + VERSION_SUFFIX_INPUT: ${{ inputs['version-suffix'] }} + run: | + # Convert "3.10,3.11,3.12" to ["3.10","3.11","3.12"] + python_array=$(echo "[\"$PYTHON_VERSIONS_INPUT\"]" | sed 's/,/","/g') + echo "python-versions=$python_array" >> $GITHUB_OUTPUT + + # Convert "3.10,3.11,3.12" to "cp310-* cp311-* cp312-*" + cibw=$(echo "$PYTHON_VERSIONS_INPUT" | sed 's/\.//g; s/,/ /g; s/\([0-9]*\)/cp\1-*/g') + echo "cibw-build=$cibw" >> $GITHUB_OUTPUT + + # Full OS list requested, e.g. ["windows-latest","ubuntu-latest",...] + os_array_all=$(echo "[\"$OS_LIST_INPUT\"]" | sed 's/,/","/g') + echo "os-list-all=$os_array_all" >> $GITHUB_OUTPUT + + # Unix-only subset (the cibuildwheel job never builds windows-latest) + unix_list=$(echo ",$OS_LIST_INPUT," | sed 's/,windows-latest,/,/g; s/^,//; s/,$//') + if [[ -z "$unix_list" ]]; then + echo "os-list-unix=[]" >> $GITHUB_OUTPUT + echo "build-unix=false" >> $GITHUB_OUTPUT + else + os_array_unix=$(echo "[\"$unix_list\"]" | sed 's/,/","/g') + echo "os-list-unix=$os_array_unix" >> $GITHUB_OUTPUT + echo "build-unix=true" >> $GITHUB_OUTPUT + fi + + if echo ",$OS_LIST_INPUT," | grep -q ",windows-latest,"; then + echo "build-windows=true" >> $GITHUB_OUTPUT + else + echo "build-windows=false" >> $GITHUB_OUTPUT + fi + + # An explicit version-suffix always wins. Otherwise, testpypi releases are + # immutable after 14 days, so auto-generate a unique dev version for every + # testpypi dry-run to avoid ever colliding with an old one. + if [[ -n "$VERSION_SUFFIX_INPUT" ]]; then + echo "version-suffix=$VERSION_SUFFIX_INPUT" >> $GITHUB_OUTPUT + elif [[ "${{ inputs.target }}" == "testpypi" ]]; then + echo "version-suffix=dev${{ github.run_id }}" >> $GITHUB_OUTPUT + else + echo "version-suffix=" >> $GITHUB_OUTPUT + fi + + build_wheels_windows: + name: Build wheels on Windows + needs: setup + if: needs.setup.outputs.build-windows == 'true' + runs-on: windows-latest + strategy: + fail-fast: False + matrix: + python-version: ${{ fromJSON(needs.setup.outputs.python-versions) }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: true + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Build wheels + env: + NLOPT_VERSION_SUFFIX: ${{ needs.setup.outputs.version-suffix }} + run: | + python -m pip install -U pip numpy swig wheel setuptools + python setup.py bdist_wheel -d dist + ls -al ./dist + + - name: Place wheels in artifacts folder + uses: actions/upload-artifact@v4 + with: + name: windows-${{ matrix.python-version }} + path: ./dist/*.whl + + build_wheels_unix: + name: Build wheels on ${{ matrix.os }} + needs: setup + if: needs.setup.outputs.build-unix == 'true' + runs-on: ${{ matrix.os }} + strategy: + fail-fast: False + matrix: + os: ${{ fromJSON(needs.setup.outputs.os-list-unix) }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: true + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Build wheels + env: + # only build CPython-3.10+ and skip 32-bit builds + CIBW_BUILD: ${{ needs.setup.outputs.cibw-build }} + CIBW_SKIP: "*-manylinux_i686 *-musllinux*" + # use latest build; manylinux_2_28 ships gcc>=12 (manylinux2014's + # gcc 10.2 is too old for numpy>=2.5 source builds) + CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux_2_28_x86_64 + CIBW_MANYLINUX_AARCH64_IMAGE: quay.io/pypa/manylinux_2_28_aarch64 + CIBW_BEFORE_ALL_MACOS: brew install swig + CIBW_BEFORE_BUILD: pip install numpy swig + NLOPT_VERSION_SUFFIX: ${{ needs.setup.outputs.version-suffix }} + # forward this host env var into the manylinux container (macOS builds run + # directly on the host, so no forwarding is needed there) + CIBW_ENVIRONMENT_PASS_LINUX: NLOPT_VERSION_SUFFIX + run: | + python -m pip install -U pip cibuildwheel + python -m cibuildwheel --output-dir dist + ls -R dist + + - name: Place wheels in artifacts folder + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.os }} + path: ./dist/*.whl + + test-wheels: + name: Test wheels + needs: [ setup, build_wheels_windows, build_wheels_unix ] + if: | + always() && + needs.setup.result == 'success' && + (needs.build_wheels_windows.result == 'success' || needs.build_wheels_windows.result == 'skipped') && + (needs.build_wheels_unix.result == 'success' || needs.build_wheels_unix.result == 'skipped') + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ${{ fromJSON(needs.setup.outputs.os-list-all) }} + python-version: ${{ fromJSON(needs.setup.outputs.python-versions) }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: true + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Retrieve packages + uses: actions/download-artifact@v4 + with: + path: dist + + - name: List items + run: | + ls -alR dist + + - name: Test Package Installation + run: | + python -m pip install --upgrade pip + + # list all files in the dist folder + ls -R dist + + # finds path to the right wheel or source file to install later + os=$(echo ${{ runner.os }} | awk '{print tolower($0)}' | head -c3) + + # version refers to the python version. So 3.8 -> 38 + version=$(echo ${{ matrix.python-version }} | sed 's/\.//g') + + if [[ "${{ matrix.os }}" == "windows-latest" ]]; then + chunk="windows-*" + else + chunk="${{matrix.os}}" + fi + + echo "Runner OS: ${{ runner.os }}" + echo "os : ${os}" + echo "version : ${version}" + echo "chunk : ${chunk}" + + # this finds files like + # nlopt-2.6.2-cp36-cp36m-win_amd64.whl + # nlopt-2.6.2-cp36-cp36m-manylinux10_amd64.whl + # nlopt-2.7.1-cp310-cp310-win_amd64.whl + file=$(find dist/${chunk} -name "nlopt-*${version}*${os}*.whl" -type f); + + echo "Installing file: ${file}" + + pip install ${file} + python extern/nlopt/test/t_python.py + + deploy: + name: Deploy packages + runs-on: ubuntu-latest + needs: test-wheels + # Fork PRs never get an OIDC token from GitHub (ACTIONS_ID_TOKEN_REQUEST_TOKEN is + # withheld regardless of the permissions below), so trusted publishing would fail + # outright for them - skip deploy rather than let it error out. + if: inputs.target != 'none' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + environment: + name: ${{ inputs.target == 'pypi' && 'prod' || 'dev' }} + permissions: + id-token: write + contents: read + + steps: + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Retrieve packages + uses: actions/download-artifact@v4 + with: + path: dist + + - name: Install twine + run: | + pip install twine + pip install -U packaging + + - name: Move files to top level directory + run: | + ls -ltR dist + python - << EOF + from pathlib import Path + import shutil + + d = Path('dist') + for f in d.rglob('*.whl'): + shutil.move(f, d / f.name) + + for f in d.iterdir(): + if f.is_dir(): + shutil.rmtree(f) + + EOF + + - name: Upload packages to testpypi + if: inputs.target == 'testpypi' + run: python -m twine upload --skip-existing --repository testpypi dist/* --verbose + + - name: Upload packages to pypi + if: inputs.target == 'pypi' + run: python -m twine upload --skip-existing dist/* --verbose diff --git a/setup.py b/setup.py index 93929c5..c8762f0 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,4 @@ +import os import re from pathlib import Path @@ -16,6 +17,12 @@ version.append(m.group(1)) version = ".".join(version) +# CI sets this to a unique dev build number (e.g. "dev12345") for testpypi dry-run +# publishes, since testpypi rejects new files added to a release once it's >14 days old. +version_suffix = os.environ.get("NLOPT_VERSION_SUFFIX") +if version_suffix: + version = f"{version}.{version_suffix}" + setup( version=version,