Switch hosting of devel packages from Google cloud to Github releases (…

…#6784)

New pre-release devel-main will be continuously updated with the devel packages for the main branch.
Also publish devel versions of viewer app.
This is to reduce Google cloud cost.

.github/workflows/documentation.yml

@@ -17,9 +17,6 @@ concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true
 
-env:
- GCE_CLI_GHA_VERSION: '416.0.0' # Fixed to avoid dependency on API changes
-
 jobs:
  headless-docs:
  # Build headless and docs
@@ -58,7 +55,7 @@ jobs:
  - name: Set up Python version
  uses: actions/setup-python@v5
  with:
- python-version: '3.8'
+ python-version: '3.11'
 
  - name: Install dependencies
  env:
@@ -87,46 +84,44 @@ jobs:
  path: docs/_out/html
  if-no-files-found: error
 
- - name: GCloud CLI auth
- if: ${{ github.ref == 'refs/heads/main' }}
- uses: 'google-github-actions/auth@v2'
- with:
- project_id: ${{ secrets.GCE_PROJECT }}
- credentials_json: '${{ secrets.GCE_SA_KEY_GPU_CI }}'
- - name: GCloud CLI setup
- if: ${{ github.ref == 'refs/heads/main' }}
- uses: google-github-actions/setup-gcloud@v2
- with:
- version: ${{ env.GCE_CLI_GHA_VERSION }}
- project_id: ${{ secrets.GCE_PROJECT }}
-
- - name: Deploy docs
+ - name: Deploy docs if all artifacts available
  if: ${{ github.ref == 'refs/heads/main' }}
+ env:
+ GH_TOKEN: ${{ github.token }}
  run: |
- # Compress and upload the docs, only for main branch
- docs_out_dir="docs/_out" # Docs in ${docs_out_dir}/html
- tar_file="${{ github.sha }}_ready.tar.gz"
+ tar_file="open3d-${GITHUB_SHA}-docs.tar.gz"
  rm -rf ${tar_file}
- tar -C ${docs_out_dir} -czvf ${tar_file} html
- gsutil cp ${tar_file} gs://open3d-docs/${tar_file}
- echo "Docs archive uploaded to:"
- echo "https://storage.googleapis.com/open3d-docs/${tar_file}"
+ # Docs in docs/_out/html
+ tar -C docs/_out -czf ${tar_file} html
 
- - name: Check wheels and ready documentation archive
- if: ${{ github.ref == 'refs/heads/main' }}
- run: |
- if [ $(gsutil ls gs://open3d-docs/${{ github.sha }}_ready* | wc -l)\
- -eq 4 ]; then
- echo "All wheels and docs available. Making docs ready."
- # Remove all marker files: Note _ at end of pattern.
- gsutil rm gs://open3d-docs/${{ github.sha }}_ready_*
- # Rename docs archive:
- gsutil mv gs://open3d-docs/${{ github.sha }}_ready.tar.gz \
- gs://open3d-docs/${{ github.sha }}.tar.gz
- # Set holds on new artifacts, release on old
- gsutil retention temp release gs://open3d-releases/*
- gsutil retention temp set gs://open3d-releases/python-wheels/*${GITHUB_SHA:0:7}*.whl
- gsutil retention temp set gs://open3d-releases/devel/*${GITHUB_SHA:0:7}*
- else
- echo "All wheels / docs not available yet."
- fi
+ echo "Waiting for other release assets..."
+ this_sha=$(echo ${GITHUB_SHA} | cut -c 1-6)
+ n_this_sha_assets=$(gh release view main-devel --json assets --jq ".assets | map(select(.name | contains(\"${this_sha}\"))) | length")
+ # Total assets from each main branch commmit:
+ # Python wheels (4x4) + Viewer (3) + C++ libs (4+2+2) = 27,
+ while ((n_this_sha_assets < 27)); do
+ sleep 60
+ echo -n "."
+ n_this_sha_assets=$(gh release view main-devel --json assets --jq ".assets | map(select(.name | contains(\"${this_sha}\"))) | length")
+ done
+ gh release upload main-devel ${tar_file} --clobber
+ gh release view main-devel
+
+ echo "\nAll assets ready. Removing release assets except from last 3 commits: ${last_shas[@]}"
+ release_assets=($(gh release view main-devel --json assets --jq '.assets[] | .name'))
+ last_shas=($(git log --pretty=format:%h --abbrev-commit -n 3))
+ echo "Waiting for all assets to be ready..."
+ for relass in "${release_assets[@]}"; do
+ found=false
+ for last_sha in "${last_shas[@]}"; do
+ if [[ $relass == *${last_sha}* ]]; then
+ found=true
+ fi
+ done
+ if [ $found == false ]; then
+ set -x
+ gh release delete-asset main-devel $relass
+ set +x
+ fi
+ done
+ gh release view main-devel

.github/workflows/macos.yml

@@ -22,7 +22,6 @@ env:
  # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
  NPROC: 4
  DEVELOPER_BUILD: ${{ github.event.inputs.developer_build || 'ON' }}
- GCE_CLI_GHA_VERSION: '416.0.0' # Fixed to avoid dependency on API changes
 
 jobs:
  MacOS:
@@ -104,11 +103,12 @@ jobs:
  - name: Build Open3D viewer app
  if: ${{ env.BUILD_SHARED_LIBS == 'OFF' }}
  run: |
+ OPEN3D_VERSION_FULL="$(grep -F OPEN3D_VERSION_FULL build/CMakeCache.txt | cut -f2 -d'=')"
  PATH=/usr/local/var/homebrew/linked/ccache/libexec:$PATH
  pushd build
  make -j${NPROC} Open3DViewer
  pushd bin
- zip -rv open3d-app-macosx-10_15-${{ runner.arch}}.zip Open3D.app
+ zip -rv "open3d-${OPEN3D_VERSION_FULL}-app-macosx-10_15-${{ runner.arch }}.zip" Open3D.app
  ccache -s
 
  - name: Upload package
@@ -119,38 +119,29 @@ jobs:
  path: build/package/${{ env.DEVEL_PKG_NAME }}
  if-no-files-found: error
 
- - name: GCloud CLI auth
- if: ${{ github.ref == 'refs/heads/main' && env.BUILD_SHARED_LIBS == 'ON' }}
- uses: 'google-github-actions/auth@v2'
- with:
- project_id: ${{ secrets.GCE_PROJECT }}
- credentials_json: '${{ secrets.GCE_SA_KEY_GPU_CI }}'
- - name: GCloud CLI setup
- if: ${{ github.ref == 'refs/heads/main' && env.BUILD_SHARED_LIBS == 'ON' }}
- uses: google-github-actions/setup-gcloud@v2
- with:
- version: ${{ env.GCE_CLI_GHA_VERSION }}
- project_id: ${{ secrets.GCE_PROJECT }}
-
- - name: Upload package to GCS bucket
+ - name: Update package devel release
  if: ${{ github.ref == 'refs/heads/main' && env.BUILD_SHARED_LIBS == 'ON' }}
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  run: |
- gsutil cp build/package/${{ env.DEVEL_PKG_NAME }} gs://open3d-releases/devel/
- echo "Download devel package at: https://storage.googleapis.com/open3d-releases/devel/${{ env.DEVEL_PKG_NAME }}"
+ gh release upload main-devel build/package/${{ env.DEVEL_PKG_NAME }} --clobber
+ gh release view main-devel
 
  - name: Upload Open3D viewer app
  uses: actions/upload-artifact@v4
  if: ${{ env.BUILD_SHARED_LIBS == 'OFF' }}
  with:
  name: open3d-app-macosx-10_15-${{ runner.arch}}
- path: build/bin/open3d-app-macosx-10_15-${{ runner.arch}}.zip
+ path: build/bin/open3d-*-app-macosx-10_15-${{ runner.arch }}.zip
  if-no-files-found: error
 
  fuse-viewer:
  name: Fuse x64 and ARM64 viewer app
  runs-on: [macos-12]
  needs: [MacOS]
  steps:
+ - name: Checkout source code # for gh release upload
+ uses: actions/checkout@v4
  - name: Download viewer apps
  uses: actions/download-artifact@v4
  with:
@@ -159,22 +150,35 @@ jobs:
 
  - name: Fuse x64 and arm64 viewer apps
  run: |
- unzip open3d-app-macosx-10_15-X64.zip -d x64
- unzip open3d-app-macosx-10_15-ARM64.zip -d arm64
+ unzip open3d-*-app-macosx-10_15-X64.zip -d x64
+ unzip open3d-*-app-macosx-10_15-ARM64.zip -d arm64
+ APP_NAME=$(ls open3d-*-app-macosx-10_15-X64.zip)
+ APP_NAME=${APP_NAME/-X64/-universal2} # includes version
  for i in arm64/Open3D.app/Contents/MacOS/*; do
  filepath=Open3D.app/Contents/MacOS/$(basename $i)
  lipo -create arm64/${filepath} x64/${filepath} -output arm64/${filepath}
  done
  mv arm64/Open3D.app Open3D.app
- zip -rv open3d-app-macosx-10_15-universal2.zip Open3D.app
+ zip -rv "${APP_NAME}" Open3D.app
 
  - name: Upload Open3D viewer app
  uses: actions/upload-artifact@v4
  with:
  name: open3d-app-macosx-10_15-universal2
- path: open3d-app-macosx-10_15-universal2.zip
+ path: open3d-*-app-macosx-10_15-universal2.zip
  if-no-files-found: error
 
+ - name: Checkout source code
+ uses: actions/checkout@v4
+ - name: Update viewer devel release
+ if: ${{ github.ref == 'refs/heads/main' }}
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ gh release upload main-devel open3d-*-app-macosx-10_15-universal2.zip --clobber
+ gh release view main-devel
+
+
  build-wheel:
  name: Build wheel
  runs-on: ${{ matrix.os }}
@@ -282,28 +286,6 @@ jobs:
  path: build/lib/python_package/pip_package/${{ env.PIP_PKG_NAME }}
  if-no-files-found: error
 
- - name: GCloud CLI auth
- if: ${{ github.ref == 'refs/heads/main' }}
- uses: 'google-github-actions/auth@v2'
- with:
- project_id: ${{ secrets.GCE_PROJECT }}
- credentials_json: '${{ secrets.GCE_SA_KEY_GPU_CI }}'
- - name: GCloud CLI setup
- if: ${{ github.ref == 'refs/heads/main' }}
- uses: google-github-actions/setup-gcloud@v2
- with:
- version: ${{ env.GCE_CLI_GHA_VERSION }}
- project_id: ${{ secrets.GCE_PROJECT }}
-
- - name: Upload wheel to GCS bucket
- if: ${{ github.ref == 'refs/heads/main' }}
- env:
- python_version: ${{ matrix.python_version }}
- run: |
- PYTAG="-cp$(echo ${{ env.python_version }} | tr -d '.')"
- gsutil cp build/lib/python_package/pip_package/${{ env.PIP_PKG_NAME }} gs://open3d-releases/python-wheels/
- echo "Download pip package at: https://storage.googleapis.com/open3d-releases/python-wheels/${{ env.PIP_PKG_NAME }}"
-
  fuse-wheel:
  name: Fuse universal2 wheel
  runs-on: [macos-12]
@@ -319,66 +301,56 @@ jobs:
  - is_main: false
  python_version: '3.10'
  steps:
+ - name: Checkout source code # for gh release upload
+ uses: actions/checkout@v4
  - name: Set up Python
  uses: actions/setup-python@v5
  with:
  python-version: ${{ matrix.python_version }}
+
  - name: Download X64 wheels
  uses: actions/download-artifact@v4
  with:
  pattern: open3d-*macosx*_x86_64.whl
  path: x64_wheels
  merge-multiple: true
+
  - name: Download ARM64 wheels
  uses: actions/download-artifact@v4
  with:
  pattern: open3d-*macosx*_arm64.whl
  path: arm64_wheels
  merge-multiple: true
+
  - name: Fuse x64 and ARM64 wheels
  env:
  python_version: ${{ matrix.python_version }}
  run: |
  PYTAG="-cp$(echo ${{ env.python_version }} | tr -d '.')"
  mkdir universal_wheels
- 
  pip install delocate
  delocate-fuse -v x64_wheels/open3d-*${PYTAG}*.whl arm64_wheels/open3d-*${PYTAG}*.whl
-
  # Normalize file name as delocate-fuse doesn't update it
  OLD_WHL_NAME=$(basename x64_wheels/open3d-*${PYTAG}*.whl)
  NEW_WHL_NAME=${OLD_WHL_NAME/x86_64/universal2}
  mv x64_wheels/${OLD_WHL_NAME} universal_wheels/${NEW_WHL_NAME}
-
  echo "PIP_PKG_NAME=$NEW_WHL_NAME" >> $GITHUB_ENV
+
  - name: Upload merged wheels
  uses: actions/upload-artifact@v4
  with:
  name: ${{ env.PIP_PKG_NAME }}
  path: universal_wheels/${{ env.PIP_PKG_NAME }}
  if-no-files-found: error
 
- - name: GCloud CLI auth
- if: ${{ github.ref == 'refs/heads/main' }}
- uses: 'google-github-actions/auth@v2'
- with:
- project_id: ${{ secrets.GCE_PROJECT }}
- credentials_json: '${{ secrets.GCE_SA_KEY_GPU_CI }}'
- - name: GCloud CLI setup
- if: ${{ github.ref == 'refs/heads/main' }}
- uses: google-github-actions/setup-gcloud@v2
- with:
- version: ${{ env.GCE_CLI_GHA_VERSION }}
- project_id: ${{ secrets.GCE_PROJECT }}
-
- - name: Upload wheel to GCS bucket
+ - name: Update devel release
  if: ${{ github.ref == 'refs/heads/main' }}
  env:
- python_version: ${{ matrix.python_version }}
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  run: |
- gsutil cp universal_wheels/${{ env.PIP_PKG_NAME }} gs://open3d-releases/python-wheels/
- echo "Download pip package at: https://storage.googleapis.com/open3d-releases/python-wheels/${{ env.PIP_PKG_NAME }}"
- 
+ gh release upload main-devel universal_wheels/${{ env.PIP_PKG_NAME }} --clobber
+ gh release view main-devel
+
  test-wheel:
  name: Test wheel
  runs-on: ${{ matrix.os }}
@@ -432,45 +404,8 @@ jobs:
  pi_tag=$(python -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')")
  test_wheel open3d*-"$pi_tag"-*_$(uname -m).whl
 
- - name: Run Python unit tests (benchmarks)
+ - name: Run Python unit tests
  run: |
  source util/ci_utils.sh
  echo "Running Open3D python tests..."
  run_python_tests
-
- ready-docs:
- name: Ready docs archive
- # no need to run on macOS
- runs-on: ubuntu-latest
- if: ${{ github.ref == 'refs/heads/main' }}
- needs: [fuse-wheel, MacOS]
- steps:
- - name: GCloud CLI auth
- uses: 'google-github-actions/auth@v2'
- with:
- project_id: ${{ secrets.GCE_PROJECT }}
- credentials_json: '${{ secrets.GCE_SA_KEY_GPU_CI }}'
- - name: GCloud CLI setup
- uses: google-github-actions/setup-gcloud@v2
- with:
- version: ${{ env.GCE_CLI_GHA_VERSION }}
- project_id: ${{ secrets.GCE_PROJECT }}
- - name: Check wheels and ready documentation archive
- run: |
- touch marker_file
- gsutil cp marker_file gs://open3d-docs/${{ github.sha }}_ready_macos
- if [ $(gsutil ls gs://open3d-docs/${{ github.sha }}_ready* | wc -l)\
- -eq 4 ]; then
- echo "All wheels and docs available. Making docs ready."
- # Remove all marker files: Note _ at end of pattern.
- gsutil rm gs://open3d-docs/${{ github.sha }}_ready_*
- # Rename docs archive:
- gsutil mv gs://open3d-docs/${{ github.sha }}_ready.tar.gz \
- gs://open3d-docs/${{ github.sha }}.tar.gz
- # Set holds on new artifacts, release on old
- gsutil retention temp release gs://open3d-releases/*
- gsutil retention temp set gs://open3d-releases/python-wheels/*${GITHUB_SHA:0:7}*.whl
- gsutil retention temp set gs://open3d-releases/devel/*${GITHUB_SHA:0:7}*
- else
- echo "All wheels / docs not available yet. Docs not ready."
- fi

.github/workflows/ubuntu-cuda.yml

@@ -156,14 +156,13 @@ jobs:
  path: open3d-devel-linux*.tar.xz
  if-no-files-found: error
 
- - name: Upload package to GCS bucket
+ - name: Update devel release
  if: ${{ github.ref == 'refs/heads/main' && env.BUILD_PACKAGE == 'true' }}
+ env:
+ GH_TOKEN: ${{ github.token }}
  run: |
- gcloud compute ssh "${INSTANCE_NAME}" \
- --zone="${GCE_ZONE}" \
- --command="ls -alh \
- && gsutil cp open3d-devel-linux-*.tar.xz \
- gs://open3d-releases/devel/"
+ gh release upload main-devel open3d-devel-linux-*.tar.xz --clobber
+ gh release view main-devel
 
  - name: VM run docker
  run: |