Skip to content

Commit

Permalink
ci: try improve benchmark build time by relaxing debug info and lto
Browse files Browse the repository at this point in the history 
relates #3200
  • Loading branch information
@Boshen
Boshen committed May 8, 2024
1 parent 7363e14 commit ffdd83f
Showing 1 changed file with 120 additions and 92 deletions.
212 changes: 120 additions & 92 deletions .github/workflows/benchmark.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,19 +28,9 @@ concurrency:
cancel-in-progress: true

jobs:
benchmark:
name: Benchmark
build:
name: Build Benchmark
runs-on: ubuntu-latest
strategy:
matrix:
# Run each benchmark in own job.
# Linter benchmark is by far the slowest, so split each fixture into own job.
component: [lexer, parser, transformer, semantic, minifier, codegen_sourcemap, sourcemap]
include:
- component: linter
fixture: 0
- component: linter
fixture: 1
steps:
- name: Checkout Branch
uses: taiki-e/checkout-action@v1
Expand All @@ -51,69 +41,55 @@ jobs:
shared-key: 'benchmark'
save-cache: ${{ github.ref_name == 'main' }}

- name: Install codspeed
uses: taiki-e/install-action@v2
with:
tool: cargo-codspeed

- name: Install Node.js
uses: actions/setup-node@v4
with:
node-version: 20
registry-url: 'https://registry.npmjs.org'

- name: Start bench results interceptor server
working-directory: ./tasks/benchmark/codspeed
env:
COMPONENT: ${{ matrix.component }}
FIXTURE: ${{ matrix.fixture }}
run: |
corepack enable
pnpm install
node capture.mjs &
- name: Build benchmark
env:
RUSTFLAGS: "-C debuginfo=2 -C strip=none -g --cfg codspeed"
RUSTFLAGS: "-C debuginfo=1 -C strip=none -g --cfg codspeed"
shell: bash
run: |
cargo build --release -p oxc_benchmark --bench ${{ matrix.component }} --features codspeed
cargo build --release -p oxc_benchmark --benches --features codspeed
rm target/release/deps/*.d
mkdir -p target/codspeed/oxc_benchmark/
mv target/release/deps/${{ matrix.component }}-* target/codspeed/oxc_benchmark
rm -rf target/codspeed/oxc_benchmark/*.d
- name: Run benchmark
uses: CodSpeedHQ/action@v2
timeout-minutes: 30
env:
FIXTURE: ${{ matrix.fixture }}
with:
run: cargo codspeed run
# Dummy token for tokenless runs, to suppress logging hash of metadata JSON (see `upload.mjs`)
token: ${{ secrets.CODSPEED_TOKEN || 'dummy' }}
upload-url: http://localhost:${{ env.INTERCEPT_PORT }}/upload

- name: Upload bench data artefact
mv target/release/deps/lexer-* target/codspeed/oxc_benchmark/lexer
mv target/release/deps/parser-* target/codspeed/oxc_benchmark/parser
mv target/release/deps/transformer-* target/codspeed/oxc_benchmark/transformer
mv target/release/deps/semantic-* target/codspeed/oxc_benchmark/semantic
mv target/release/deps/minifier-* target/codspeed/oxc_benchmark/minifier
mv target/release/deps/codegen_sourcemap-* target/codspeed/oxc_benchmark/codegen_sourcemap
mv target/release/deps/sourcemap-* target/codspeed/oxc_benchmark/sourcemap
mv target/release/deps/linter-* target/codspeed/oxc_benchmark/linter
- name: Upload Binary
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.component }}${{ matrix.fixture }}
path: ${{ env.DATA_DIR }}
if-no-files-found: error
retention-days: 1
name: benchmark
path: ./target/codspeed/oxc_benchmark

benchmark-napi:
name: Benchmark NAPI parser
benchmark:
needs: build
name: Benchmark
runs-on: ubuntu-latest
if: false
strategy:
fail-fast: true
matrix:
# component: [lexer, parser, transformer, semantic, minifier, codegen_sourcemap, sourcemap, linter]
component: [lexer]

steps:
- name: Checkout Branch
uses: taiki-e/checkout-action@v1

- name: Install Rust Toolchain
uses: ./.github/actions/rustup
- name: Download Artifacts
uses: actions/download-artifact@v4
with:
shared-key: 'benchmark_napi'
save-cache: ${{ github.ref_name == 'main' }}
name: benchmark
path: ./target/codspeed/oxc_benchmark

- name: Fix permission loss
shell: bash
run: |
ls ./target/codspeed/oxc_benchmark
chmod +x ./target/codspeed/oxc_benchmark/${{ matrix.component }}
- name: Install codspeed
uses: taiki-e/install-action@v2
Expand All @@ -129,55 +105,106 @@ jobs:
- name: Start bench results interceptor server
working-directory: ./tasks/benchmark/codspeed
env:
COMPONENT: 'parser_napi'
COMPONENT: ${{ matrix.component }}
run: |
corepack enable
pnpm install
node capture.mjs &
# CodSpeed gets measurements completely off for NAPI if run in `CodSpeedHQ/action`,
# so instead run real benchmark without CodSpeed's instrumentation and save the results.
# Then "Run Rust benchmark" step below runs a loop of some simple Rust code the number
# of times required to take same amount of time as the real benchmark took.
# This is all a workaround for https://github.com/CodSpeedHQ/action/issues/96
- name: Build NAPI Benchmark
working-directory: ./napi/parser
run: |
corepack enable
pnpm install
pnpm build
- name: Run NAPI Benchmark
working-directory: ./napi/parser
run: node parse.bench.mjs

- name: Build Rust benchmark
env:
RUSTFLAGS: "-C debuginfo=2 -C strip=none -g --cfg codspeed"
shell: bash
run: |
cargo build --release -p oxc_benchmark --bench parser_napi --no-default-features --features codspeed_napi
mkdir -p target/codspeed/oxc_benchmark/
mv target/release/deps/parser_napi-* target/codspeed/oxc_benchmark
rm -rf target/codspeed/oxc_benchmark/*.d
- name: Run Rust benchmark
- name: Run benchmark
uses: CodSpeedHQ/action@v2
timeout-minutes: 30
with:
run: cargo codspeed run
# Dummy token for tokenless runs, to suppress logging hash of metadata JSON (see `upload.mjs`)
token: ${{ secrets.CODSPEED_TOKEN || 'dummy' }}
upload-url: http://localhost:${{ env.INTERCEPT_PORT }}/upload
run: cargo codspeed run ${{ matrix.component }}

- name: Upload bench data artefact
uses: actions/upload-artifact@v4
with:
name: 'parser_napi'
path: ${{ env.DATA_DIR }}
name: result-${{ matrix.component }}
path: ${{ env.DATA_DIR }} # # env.DEFAULT_PORT from capture.mjs
if-no-files-found: error
retention-days: 1

# benchmark-napi:
# name: Benchmark NAPI parser
# runs-on: ubuntu-latest
# if: false
# steps:
# - name: Checkout Branch
# uses: taiki-e/checkout-action@v1

# - name: Install Rust Toolchain
# uses: ./.github/actions/rustup
# with:
# shared-key: 'benchmark_napi'
# save-cache: ${{ github.ref_name == 'main' }}

# - name: Install codspeed
# uses: taiki-e/install-action@v2
# with:
# tool: cargo-codspeed

# - name: Install Node.js
# uses: actions/setup-node@v4
# with:
# node-version: 20
# registry-url: 'https://registry.npmjs.org'

# - name: Start bench results interceptor server
# working-directory: ./tasks/benchmark/codspeed
# env:
# COMPONENT: 'parser_napi'
# run: |
# corepack enable
# pnpm install
# node capture.mjs &

# # CodSpeed gets measurements completely off for NAPI if run in `CodSpeedHQ/action`,
# # so instead run real benchmark without CodSpeed's instrumentation and save the results.
# # Then "Run Rust benchmark" step below runs a loop of some simple Rust code the number
# # of times required to take same amount of time as the real benchmark took.
# # This is all a workaround for https://github.com/CodSpeedHQ/action/issues/96
# - name: Build NAPI Benchmark
# working-directory: ./napi/parser
# run: |
# corepack enable
# pnpm install
# pnpm build

# - name: Run NAPI Benchmark
# working-directory: ./napi/parser
# run: node parse.bench.mjs

# - name: Build Rust benchmark
# env:
# RUSTFLAGS: "-C debuginfo=2 -C strip=none -g --cfg codspeed"
# shell: bash
# run: |
# cargo build --release -p oxc_benchmark --bench parser_napi --no-default-features --features codspeed_napi
# mkdir -p target/codspeed/oxc_benchmark/
# mv target/release/deps/parser_napi-* target/codspeed/oxc_benchmark
# rm -rf target/codspeed/oxc_benchmark/*.d

# - name: Run Rust benchmark
# uses: CodSpeedHQ/action@v2
# timeout-minutes: 30
# with:
# run: cargo codspeed run
# # Dummy token for tokenless runs, to suppress logging hash of metadata JSON (see `upload.mjs`)
# token: ${{ secrets.CODSPEED_TOKEN || 'dummy' }}
# upload-url: http://localhost:${{ env.INTERCEPT_PORT }}/upload

# - name: Upload bench data artefact
# uses: actions/upload-artifact@v4
# with:
# name: 'parser_napi'
# path: ${{ env.DATA_DIR }}
# if-no-files-found: error
# retention-days: 1

upload:
name: Upload benchmarks
# needs: [benchmark, benchmark-napi]
Expand All @@ -203,8 +230,9 @@ jobs:
- name: Download artefacts
uses: actions/download-artifact@v4
with:
path: ${{ env.DATA_DIR }}
merge-multiple: true
pattern: result-*
path: ${{ env.DATA_DIR }} # env.DEFAULT_PORT from create_temp_dir.mjs

- name: Upload to Codspeed
working-directory: ./tasks/benchmark/codspeed
Expand Down

0 comments on commit ffdd83f

Please sign in to comment.