feat: Include waved binary in Conda distributions #2266

.github/workflows/publish-conda.yml

@@ -8,7 +8,6 @@ on:
  required: true
 
 env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  VERSION: "${{ github.event.inputs.version }}"
 
 jobs:
@@ -20,13 +19,39 @@ jobs:
  with:
  token: ${{ secrets.GIT_TOKEN }}
 
- - name: Publish to Conda
- uses: marek-mihok/[email protected]
+ - uses: actions/setup-go@v1
  with:
- CondaDir: 'py/h2o_wave/conda'
- Platforms: 'noarch'
- CondaUsername: ${{ secrets.CONDA_USERNAME }}
- CondaPassword: ${{ secrets.CONDA_PASSWORD }}
+ go-version: '1.19.4'
+
+ - uses: actions/setup-node@v3
+ with:
+ node-version-file: '.nvmrc'
+
+ - name: Setup
+ run: make setup
+
+ - uses: r-lib/actions/setup-r@v2
+ - uses: r-lib/actions/setup-r-dependencies@v2
+ with:
+ working-directory: ./r
+
+ - name: Setup Miniconda
+ uses: conda-incubator/setup-miniconda@v3
+ with:
+ auto-activate-base: false
+ activate-environment: ""
+ conda-build-version: '24.3.0'
+
+ - name: Build Release
+ run: make release
  env:
  # VERSION clashes with conda build. Use PKG_VERSION instead.
  PKG_VERSION: ${{ env.VERSION }}
+ NODE_OPTIONS: '--max-old-space-size=8192'
+
+ - name: Publish to Conda
+ uses: marek-mihok/[email protected]
+ with:
+ PackagesDir: 'py/h2o_wave/sdist'
+ CondaUsername: ${{ secrets.CONDA_USERNAME }}
+ CondaPassword: ${{ secrets.CONDA_PASSWORD }}

.github/workflows/release-wave.yml

@@ -41,9 +41,18 @@ jobs:
  with:
  working-directory: ./r
 
+ - name: Setup Miniconda
+ uses: conda-incubator/setup-miniconda@v3
+ with:
+ auto-activate-base: false
+ activate-environment: ""
+ conda-build-version: '24.3.0'
+
  - name: Build Release
  run: make release
  env:
+ # VERSION clashes with conda build. Use PKG_VERSION instead.
+ PKG_VERSION: ${{ env.VERSION }}
  NODE_OPTIONS: '--max-old-space-size=8192'
 
  - name: Unit Test
@@ -79,15 +88,11 @@ jobs:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
  - name: Publish to Conda
- uses: marek-mihok/publish_conda_package_action@v1.0.1
+ uses: marek-mihok/upload_to_anaconda_cloud_action@v1.0.0
  with:
- CondaDir: 'py/h2o_wave/conda'
- Platforms: 'noarch'
+ PackagesDir: 'py/h2o_wave/sdist'
  CondaUsername: ${{ secrets.CONDA_USERNAME }}
  CondaPassword: ${{ secrets.CONDA_PASSWORD }}
- env:
- # VERSION clashes with conda build. Use PKG_VERSION instead.
- PKG_VERSION: ${{ env.VERSION }}
 
  - name: Publish to PyPI
  uses: pypa/gh-action-pypi-publish@master

py/h2o_wave/Makefile

@@ -1,3 +1,19 @@
+build_conda_package = \
+ if [ "$(1)" != "noarch" ]; then \
+ mkdir -p conda/temp; \
+ cp -a ../../build/wave-$(VERSION)-$(1)/. conda/temp; \
+ NO_ARCH=0 conda build --output-folder temp_build/ conda; \
+ find temp_build/ -name *.tar.bz2 | while read -r file; do \
+ conda convert --force --platform $(2) "$$file" -o temp_build/; \
+ done \
+ fi && \
+ if [ "$(1)" == "noarch" ]; then \
+ NO_ARCH=1 conda build --output-folder temp_build/ conda; \
+ fi && \
+ mkdir -p sdist/$(2) && \
+ mv temp_build/$(2)/*.tar.bz2 sdist/$(2) && \
+ rm -rf temp_build conda/temp
+
 .PHONY: build
 build: purge
  H2O_WAVE_PLATFORM=win_amd64 ../venv/bin/python3 -m build --wheel
@@ -6,7 +22,11 @@ build: purge
  H2O_WAVE_PLATFORM=macosx_11_0_arm64 ../venv/bin/python3 -m build --wheel
  H2O_WAVE_PLATFORM=macosx_12_0_arm64 ../venv/bin/python3 -m build --wheel
  ../venv/bin/python3 -m build --wheel
+ $(call build_conda_package,darwin-arm64,osx-arm64)
+ $(call build_conda_package,darwin-amd64,osx-64)
+ $(call build_conda_package,linux-amd64,linux-64)
+ $(call build_conda_package,windows-amd64,win-64)
+ $(call build_conda_package,noarch,noarch)
 
 purge: ## Purge previous build
- rm -f h2o_wave/metadata.py
- rm -rf build dist h2o_wave.egg-info
+ rm -rf build dist sdist h2o_wave.egg-info temp_build conda/temp h2o_wave/metadata.py

py/h2o_wave/conda/conda_build_config.yaml

@@ -0,0 +1,8 @@
+# TODO: Build a single package for all python versions once this is resolved: https://github.com/conda/conda-build/issues/2611#issuecomment-1239338538 
+python_version:
+ - 3.8
+ - 3.9
+ - 3.10
+ - 3.11
+ - 3.12
+

py/h2o_wave/conda/meta.yaml

@@ -15,20 +15,33 @@ about:
 
 requirements:
  build: 
+ {% if NO_ARCH == "0" %}
+ - python={{ python_version }}
+ {% endif %}
  {% for dep in pyproject['build-system']['requires'] %}
  - {{ dep.lower() }}
  {% endfor %}
  run:
+ {% if NO_ARCH == "0" %}
+ - python={{ python_version }}
+ {% endif %}
  {% for dep in project['dependencies'] %}
  - {{ dep.lower() }}
  {% endfor %}
 source:
  path: ..
 
 build:
- noarch: python
  script: python -m pip install . -vv
+ string: py{{ python_version | replace(".", "") }}
  entry_points:
  {% for module, entrypoints in project['scripts'].items() %}
  - {{ module }} = {{ entrypoints }}
- {% endfor %}
+ {% endfor %}
+ include_recipe: False
+ script_env:
+ - NO_ARCH
+ {% if NO_ARCH == "1" %}
+ noarch: python
+ string: "0"
+ {% endif %}

py/h2o_wave/hatch_build.py

@@ -26,6 +26,10 @@ def initialize(self, _version, build_data):
  if not platform:
  # Create a default metadata file in case of noarch builds.
  create_metadata_file('linux', 'amd64')
+ # If conda build, copy binaries into package.
+ binaries_path = os.path.join('conda', 'temp')
+ if os.environ.get('CONDA_BUILD') and os.path.exists(binaries_path):
+ self.copy_binaries_into_package(binaries_path)
  return
 
  build_data['tag'] = f'py3-none-{platform}'
@@ -42,12 +46,7 @@ def initialize(self, _version, build_data):
  elif platform == 'manylinux1_x86_64':
  operating_system = 'linux'
 
- binaries_path = os.path.join('..', '..', 'build', f'wave-{version}-{operating_system}-{arch}')
- if not os.path.exists(binaries_path):
- raise Exception(f'{binaries_path} does not exist. Run make release first to generate server binaries.')
-
- self.copy_files(binaries_path, 'tmp', ['demo', 'examples', 'test'])
- self.copy_files('project_templates', 'tmp', [], True)
+ self.copy_binaries_into_package(os.path.join('..', '..', 'build', f'wave-{version}-{operating_system}-{arch}'))
 
  create_metadata_file(operating_system, arch)
 
@@ -61,5 +60,13 @@ def copy_files(self, src, dst, ignore, keep_dir=False) -> None:
  elif os.path.isdir(src_file) and file_name not in ignore:
  self.copy_files(src_file, dst_file, ignore)
 
+ def copy_binaries_into_package(self, binaries_path):
+ if not os.path.exists(binaries_path):
+ raise Exception(f'{binaries_path} does not exist. Run make release first to generate server binaries.')
+
+ self.copy_files(binaries_path, 'tmp', ['demo', 'examples', 'test'])
+ self.copy_files('project_templates', 'tmp', [], True)
+
  def finalize(self, version: str, build_data: Dict[str, Any], artifact_path: str) -> None:
  shutil.rmtree('tmp', ignore_errors=True)
+

website/docs/installation.mdx

@@ -22,9 +22,6 @@ pip install h2o-wave
 conda config --append channels conda-forge
 conda install -c h2oai h2o-wave
 ```
-:::info
-Conda packaging does not contain Wave server which means, you will still need to run the wave server (waved) [separately](/docs/tutorial-hello#step-1-start-the-wave-server).
-:::
 
 </TabItem>
 
@@ -64,9 +61,6 @@ conda config --append channels conda-forge
 conda install -c h2oai h2o-wave
 ```
 
-:::info
-Conda packaging does not contain Wave server which means, you will still need to run the wave server (waved) [separately](/docs/tutorial-hello#step-1-start-the-wave-server).
-:::
 
 </TabItem>
 </Tabs>