CICD github action build publish pypi、Release Tag (#3886)

* 测试用例

* CICD 流程

* CICD 流程

* CICD 流程

.github/actions/poetry_setup/action.yml

@@ -0,0 +1,93 @@
+# An action for setting up poetry install with caching.
+# Using a custom action since the default action does not
+# take poetry install groups into account.
+# Action code from:
+# https://github.com/actions/setup-python/issues/505#issuecomment-1273013236
+name: poetry-install-with-caching
+description: Poetry install with support for caching of dependency groups.
+
+inputs:
+ python-version:
+ description: Python version, supporting MAJOR.MINOR only
+ required: true
+
+ poetry-version:
+ description: Poetry version
+ required: true
+
+ cache-key:
+ description: Cache key to use for manual handling of caching
+ required: true
+
+ working-directory:
+ description: Directory whose poetry.lock file should be cached
+ required: true
+
+runs:
+ using: composite
+ steps:
+ - uses: actions/setup-python@v5
+ name: Setup python ${{ inputs.python-version }}
+ id: setup-python
+ with:
+ python-version: ${{ inputs.python-version }}
+
+ - uses: actions/cache@v4
+ id: cache-bin-poetry
+ name: Cache Poetry binary - Python ${{ inputs.python-version }}
+ env:
+ SEGMENT_DOWNLOAD_TIMEOUT_MIN: "1"
+ with:
+ path: |
+ /opt/pipx/venvs/poetry
+ # This step caches the poetry installation, so make sure it's keyed on the poetry version as well.
+ key: bin-poetry-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-${{ inputs.poetry-version }}
+
+ - name: Refresh shell hashtable and fixup softlinks
+ if: steps.cache-bin-poetry.outputs.cache-hit == 'true'
+ shell: bash
+ env:
+ POETRY_VERSION: ${{ inputs.poetry-version }}
+ PYTHON_VERSION: ${{ inputs.python-version }}
+ run: |
+ set -eux
+
+ # Refresh the shell hashtable, to ensure correct `which` output.
+ hash -r
+
+ # `actions/cache@v3` doesn't always seem able to correctly unpack softlinks.
+ # Delete and recreate the softlinks pipx expects to have.
+ rm /opt/pipx/venvs/poetry/bin/python
+ cd /opt/pipx/venvs/poetry/bin
+ ln -s "$(which "python$PYTHON_VERSION")" python
+ chmod +x python
+ cd /opt/pipx_bin/
+ ln -s /opt/pipx/venvs/poetry/bin/poetry poetry
+ chmod +x poetry
+
+ # Ensure everything got set up correctly.
+ /opt/pipx/venvs/poetry/bin/python --version
+ /opt/pipx_bin/poetry --version
+
+ - name: Install poetry
+ if: steps.cache-bin-poetry.outputs.cache-hit != 'true'
+ shell: bash
+ env:
+ POETRY_VERSION: ${{ inputs.poetry-version }}
+ PYTHON_VERSION: ${{ inputs.python-version }}
+ # Install poetry using the python version installed by setup-python step.
+ run: pipx install "poetry==$POETRY_VERSION" --python '${{ steps.setup-python.outputs.python-path }}' --verbose
+
+ - name: Restore pip and poetry cached dependencies
+ uses: actions/cache@v4
+ env:
+ SEGMENT_DOWNLOAD_TIMEOUT_MIN: "4"
+ WORKDIR: ${{ inputs.working-directory == '' && '.' || inputs.working-directory }}
+ with:
+ path: |
+ ~/.cache/pip
+ ~/.cache/pypoetry/virtualenvs
+ ~/.cache/pypoetry/cache
+ ~/.cache/pypoetry/artifacts
+ ${{ env.WORKDIR }}/.venv
+ key: py-deps-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-poetry-${{ inputs.poetry-version }}-${{ inputs.cache-key }}-${{ hashFiles(format('{0}/**/poetry.lock', env.WORKDIR)) }}

.github/workflows/_release.yml

@@ -0,0 +1,264 @@
+name: release
+run-name: Release ${{ inputs.working-directory }} by @${{ github.actor }}
+on:
+ workflow_call:
+ branches:
+ - dev
+ inputs:
+ working-directory:
+ required: true
+ type: string
+ description: "From which folder this pipeline executes"
+ workflow_dispatch:
+ branches:
+ - dev
+ inputs:
+ working-directory:
+ required: true
+ type: string
+ default: './chatchat-server'
+ description: "From which folder this pipeline executes"
+env:
+ PYTHON_VERSION: "3.8"
+ POETRY_VERSION: "1.7.1"
+
+jobs:
+ build:
+ if: github.ref == 'refs/heads/master'
+ environment: Scheduled testing
+ runs-on: ubuntu-latest
+
+ outputs:
+ pkg-name: ${{ steps.check-version.outputs.pkg-name }}
+ version: ${{ steps.check-version.outputs.version }}
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Python + Poetry ${{ env.POETRY_VERSION }}
+ uses: "./.github/actions/poetry_setup"
+ with:
+ python-version: ${{ env.PYTHON_VERSION }}
+ poetry-version: ${{ env.POETRY_VERSION }}
+ working-directory: ${{ inputs.working-directory }}
+ cache-key: release
+
+ # We want to keep this build stage *separate* from the release stage,
+ # so that there's no sharing of permissions between them.
+ # The release stage has trusted publishing and GitHub repo contents write access,
+ # and we want to keep the scope of that access limited just to the release job.
+ # Otherwise, a malicious `build` step (e.g. via a compromised dependency)
+ # could get access to our GitHub or PyPI credentials.
+ #
+ # Per the trusted publishing GitHub Action:
+ # > It is strongly advised to separate jobs for building [...]
+ # > from the publish job.
+ # https://github.com/pypa/gh-action-pypi-publish#non-goals
+ - name: Build project for distribution
+ run: poetry build
+ working-directory: ${{ inputs.working-directory }}
+
+ - name: Upload build
+ uses: actions/upload-artifact@v4
+ with:
+ name: dist
+ path: ${{ inputs.working-directory }}/dist/
+
+ - name: Check Version
+ id: check-version
+ shell: bash
+ working-directory: ${{ inputs.working-directory }}
+ run: |
+ echo pkg-name="$(poetry version | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT
+ echo version="$(poetry version --short)" >> $GITHUB_OUTPUT
+
+ test-pypi-publish:
+ needs:
+ - build
+ uses:
+ ./.github/workflows/_test_release.yml
+ with:
+ working-directory: ${{ inputs.working-directory }}
+ secrets: inherit
+
+ pre-release-checks:
+ needs:
+ - build
+ - test-pypi-publish
+ environment: Scheduled testing
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ # We explicitly *don't* set up caching here. This ensures our tests are
+ # maximally sensitive to catching breakage.
+ #
+ # For example, here's a way that caching can cause a falsely-passing test:
+ # - Make the langchain package manifest no longer list a dependency package
+ # as a requirement. This means it won't be installed by `pip install`,
+ # and attempting to use it would cause a crash.
+ # - That dependency used to be required, so it may have been cached.
+ # When restoring the venv packages from cache, that dependency gets included.
+ # - Tests pass, because the dependency is present even though it wasn't specified.
+ # - The package is published, and it breaks on the missing dependency when
+ # used in the real world.
+
+ - name: Set up Python + Poetry ${{ env.POETRY_VERSION }}
+ uses: "./.github/actions/poetry_setup"
+ with:
+ python-version: ${{ env.PYTHON_VERSION }}
+ poetry-version: ${{ env.POETRY_VERSION }}
+ working-directory: ${{ inputs.working-directory }}
+
+ - name: Import published package
+ shell: bash
+ working-directory: ${{ inputs.working-directory }}
+ env:
+ PKG_NAME: ${{ needs.build.outputs.pkg-name }}
+ VERSION: ${{ needs.build.outputs.version }}
+ # Here we use:
+ # - The default regular PyPI index as the *primary* index, meaning
+ # that it takes priority (https://pypi.org/simple)
+ # - The test PyPI index as an extra index, so that any dependencies that
+ # are not found on test PyPI can be resolved and installed anyway.
+ # (https://test.pypi.org/simple). This will include the PKG_NAME==VERSION
+ # package because VERSION will not have been uploaded to regular PyPI yet.
+ # - attempt install again after 5 seconds if it fails because there is
+ # sometimes a delay in availability on test pypi
+ run: |
+ poetry run pip install \
+ --extra-index-url https://test.pypi.org/simple/ \
+ "$PKG_NAME==$VERSION" || \
+ ( \
+ sleep 5 && \
+ poetry run pip install \
+ --extra-index-url https://test.pypi.org/simple/ \
+ "$PKG_NAME==$VERSION" \
+ )
+
+ # Replace all dashes in the package name with underscores,
+ # since that's how Python imports packages with dashes in the name.
+ IMPORT_NAME="$(echo "$PKG_NAME" | sed s/-/_/g)"
+
+ poetry run python -c "import $IMPORT_NAME; print(dir($IMPORT_NAME))"
+
+ - name: Import test dependencies
+ run: poetry install --with test
+ working-directory: ${{ inputs.working-directory }}
+
+ # Overwrite the local version of the package with the test PyPI version.
+ - name: Import published package (again)
+ working-directory: ${{ inputs.working-directory }}
+ shell: bash
+ env:
+ PKG_NAME: ${{ needs.build.outputs.pkg-name }}
+ VERSION: ${{ needs.build.outputs.version }}
+ run: |
+ poetry run pip install \
+ --extra-index-url https://test.pypi.org/simple/ \
+ "$PKG_NAME==$VERSION"
+
+ - name: Run unit tests
+ run: make tests
+ env:
+ ZHIPUAI_API_KEY: ${{ secrets.ZHIPUAI_API_KEY }}
+ ZHIPUAI_BASE_URL: ${{ secrets.ZHIPUAI_BASE_URL }}
+ working-directory: ${{ inputs.working-directory }}
+
+# - name: Run integration tests
+# env:
+# ZHIPUAI_API_KEY: ${{ secrets.ZHIPUAI_API_KEY }}
+# ZHIPUAI_BASE_URL: ${{ secrets.ZHIPUAI_BASE_URL }}
+# run: make integration_tests
+# working-directory: ${{ inputs.working-directory }}
+
+ publish:
+ needs:
+ - build
+ - test-pypi-publish
+ - pre-release-checks
+ environment: Scheduled testing
+ runs-on: ubuntu-latest
+ permissions:
+ # This permission is used for trusted publishing:
+ # https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/
+ #
+ # Trusted publishing has to also be configured on PyPI for each package:
+ # https://docs.pypi.org/trusted-publishers/adding-a-publisher/
+ id-token: write
+
+ defaults:
+ run:
+ working-directory: ${{ inputs.working-directory }}
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Python + Poetry ${{ env.POETRY_VERSION }}
+ uses: "./.github/actions/poetry_setup"
+ with:
+ python-version: ${{ env.PYTHON_VERSION }}
+ poetry-version: ${{ env.POETRY_VERSION }}
+ working-directory: ${{ inputs.working-directory }}
+ cache-key: release
+
+ - uses: actions/download-artifact@v4
+ with:
+ name: dist
+ path: ${{ inputs.working-directory }}/dist/
+
+ - name: Publish package distributions to PyPI
+ uses: pypa/gh-action-pypi-publish@release/v1
+ with:
+ packages-dir: ${{ inputs.working-directory }}/dist/
+ verbose: true
+ print-hash: true
+ password: ${{ secrets.PYPI_API_TOKEN }}
+ # We overwrite any existing distributions with the same name and version.
+ # This is *only for CI use* and is *extremely dangerous* otherwise!
+ # https://github.com/pypa/gh-action-pypi-publish#tolerating-release-package-file-duplicates
+ skip-existing: true
+
+ mark-release:
+ needs:
+ - build
+ - test-pypi-publish
+ - pre-release-checks
+ - publish
+ environment: Scheduled testing
+ runs-on: ubuntu-latest
+ permissions:
+ # This permission is needed by `ncipollo/release-action` to
+ # create the GitHub release.
+ contents: write
+
+ defaults:
+ run:
+ working-directory: ${{ inputs.working-directory }}
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Python + Poetry ${{ env.POETRY_VERSION }}
+ uses: "./.github/actions/poetry_setup"
+ with:
+ python-version: ${{ env.PYTHON_VERSION }}
+ poetry-version: ${{ env.POETRY_VERSION }}
+ working-directory: ${{ inputs.working-directory }}
+ cache-key: release
+
+ - uses: actions/download-artifact@v4
+ with:
+ name: dist
+ path: ${{ inputs.working-directory }}/dist/
+
+ - name: Create Release
+ uses: ncipollo/release-action@v1
+ if: ${{ inputs.working-directory == './chatchat-server' }}
+ with:
+ artifacts: "dist/*"
+ token: ${{ secrets.GITHUB_TOKEN }}
+ draft: false
+ generateReleaseNotes: true
+ tag: v${{ needs.build.outputs.version }}
+ commit: main