ci: try improve benchmark build time by relaxing debug info and lto

relates #3200
oxc-project · May 8, 2024 · 15b9947 · 15b9947
1 parent 7363e14
commit 15b9947
Show file tree

Hide file tree

Showing 3 changed files with 134 additions and 103 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -14,7 +14,6 @@ on:
  push:
  branches:
  - main
- - bench-*
  paths:
  - '**/*.rs'
  - 'napi/parser/**/*.js'
@@ -28,19 +27,9 @@ concurrency:
  cancel-in-progress: true
 
 jobs:
- benchmark:
- name: Benchmark
+ build:
+ name: Build Benchmark
  runs-on: ubuntu-latest
- strategy:
- matrix:
- # Run each benchmark in own job.
- # Linter benchmark is by far the slowest, so split each fixture into own job.
- component: [lexer, parser, transformer, semantic, minifier, codegen_sourcemap, sourcemap]
- include:
- - component: linter
- fixture: 0
- - component: linter
- fixture: 1
  steps:
  - name: Checkout Branch
  uses: taiki-e/checkout-action@v1
@@ -51,69 +40,62 @@ jobs:
  shared-key: 'benchmark'
  save-cache: ${{ github.ref_name == 'main' }}
 
- - name: Install codspeed
- uses: taiki-e/install-action@v2
- with:
- tool: cargo-codspeed
-
- - name: Install Node.js
- uses: actions/setup-node@v4
- with:
- node-version: 20
- registry-url: 'https://registry.npmjs.org'
-
- - name: Start bench results interceptor server
- working-directory: ./tasks/benchmark/codspeed
- env:
- COMPONENT: ${{ matrix.component }}
- FIXTURE: ${{ matrix.fixture }}
- run: |
- corepack enable
- pnpm install
- node capture.mjs &
-
  - name: Build benchmark
  env:
- RUSTFLAGS: "-C debuginfo=2 -C strip=none -g --cfg codspeed"
+ RUSTFLAGS: "-C debuginfo=1 -C strip=none -g --cfg codspeed"
  shell: bash
  run: |
- cargo build --release -p oxc_benchmark --bench ${{ matrix.component }} --features codspeed
+ cargo build --release -p oxc_benchmark --benches --features codspeed
+ rm target/release/deps/*.d
  mkdir -p target/codspeed/oxc_benchmark/
- mv target/release/deps/${{ matrix.component }}-* target/codspeed/oxc_benchmark
- rm -rf target/codspeed/oxc_benchmark/*.d
-
- - name: Run benchmark
- uses: CodSpeedHQ/action@v2
- timeout-minutes: 30
- env:
- FIXTURE: ${{ matrix.fixture }}
- with:
- run: cargo codspeed run
- # Dummy token for tokenless runs, to suppress logging hash of metadata JSON (see `upload.mjs`)
- token: ${{ secrets.CODSPEED_TOKEN || 'dummy' }}
- upload-url: http://localhost:${{ env.INTERCEPT_PORT }}/upload
-
- - name: Upload bench data artefact
+ mv target/release/deps/lexer-* target/codspeed/oxc_benchmark/lexer
+ mv target/release/deps/parser-* target/codspeed/oxc_benchmark/parser
+ mv target/release/deps/transformer-* target/codspeed/oxc_benchmark/transformer
+ mv target/release/deps/semantic-* target/codspeed/oxc_benchmark/semantic
+ mv target/release/deps/minifier-* target/codspeed/oxc_benchmark/minifier
+ mv target/release/deps/codegen_sourcemap-* target/codspeed/oxc_benchmark/codegen_sourcemap
+ mv target/release/deps/sourcemap-* target/codspeed/oxc_benchmark/sourcemap
+ mv target/release/deps/linter-* target/codspeed/oxc_benchmark/linter
+
+ - name: Upload Binary
  uses: actions/upload-artifact@v4
  with:
- name: ${{ matrix.component }}${{ matrix.fixture }}
- path: ${{ env.DATA_DIR }}
  if-no-files-found: error
- retention-days: 1
+ name: benchmark
+ path: ./target/codspeed/oxc_benchmark
 
- benchmark-napi:
- name: Benchmark NAPI parser
+ benchmark:
+ needs: build
+ name: Benchmark
  runs-on: ubuntu-latest
- if: false
+ strategy:
+ fail-fast: true
+ matrix:
+ component:
+ - lexer
+ - parser
+ - transformer
+ - semantic
+ - minifier
+ - codegen_sourcemap
+ - sourcemap
+ - linter
+
  steps:
  - name: Checkout Branch
  uses: taiki-e/checkout-action@v1
 
- - name: Install Rust Toolchain
- uses: ./.github/actions/rustup
+ - name: Download Artifacts
+ uses: actions/download-artifact@v4
  with:
- shared-key: 'benchmark_napi'
- save-cache: ${{ github.ref_name == 'main' }}
+ name: benchmark
+ path: ./target/codspeed/oxc_benchmark
+
+ - name: Fix permission loss
+ shell: bash
+ run: |
+ ls ./target/codspeed/oxc_benchmark
+ chmod +x ./target/codspeed/oxc_benchmark/${{ matrix.component }}
 
  - name: Install codspeed
  uses: taiki-e/install-action@v2
@@ -129,55 +111,106 @@ jobs:
  - name: Start bench results interceptor server
  working-directory: ./tasks/benchmark/codspeed
  env:
- COMPONENT: 'parser_napi'
+ COMPONENT: ${{ matrix.component }}
  run: |
  corepack enable
  pnpm install
  node capture.mjs &
 
- # CodSpeed gets measurements completely off for NAPI if run in `CodSpeedHQ/action`,
- # so instead run real benchmark without CodSpeed's instrumentation and save the results.
- # Then "Run Rust benchmark" step below runs a loop of some simple Rust code the number
- # of times required to take same amount of time as the real benchmark took.
- # This is all a workaround for https://github.com/CodSpeedHQ/action/issues/96
- - name: Build NAPI Benchmark
- working-directory: ./napi/parser
- run: |
- corepack enable
- pnpm install
- pnpm build
-
- - name: Run NAPI Benchmark
- working-directory: ./napi/parser
- run: node parse.bench.mjs
-
- - name: Build Rust benchmark
- env:
- RUSTFLAGS: "-C debuginfo=2 -C strip=none -g --cfg codspeed"
- shell: bash
- run: |
- cargo build --release -p oxc_benchmark --bench parser_napi --no-default-features --features codspeed_napi
- mkdir -p target/codspeed/oxc_benchmark/
- mv target/release/deps/parser_napi-* target/codspeed/oxc_benchmark
- rm -rf target/codspeed/oxc_benchmark/*.d
-
- - name: Run Rust benchmark
+ - name: Run benchmark
  uses: CodSpeedHQ/action@v2
  timeout-minutes: 30
  with:
- run: cargo codspeed run
  # Dummy token for tokenless runs, to suppress logging hash of metadata JSON (see `upload.mjs`)
  token: ${{ secrets.CODSPEED_TOKEN || 'dummy' }}
  upload-url: http://localhost:${{ env.INTERCEPT_PORT }}/upload
+ run: cargo codspeed run ${{ matrix.component }}
 
  - name: Upload bench data artefact
  uses: actions/upload-artifact@v4
  with:
- name: 'parser_napi'
- path: ${{ env.DATA_DIR }}
+ name: result-${{ matrix.component }}
+ path: ${{ env.DATA_DIR }} # # env.DEFAULT_PORT from capture.mjs
  if-no-files-found: error
  retention-days: 1
 
+ # benchmark-napi:
+ # name: Benchmark NAPI parser
+ # runs-on: ubuntu-latest
+ # if: false
+ # steps:
+ # - name: Checkout Branch
+ # uses: taiki-e/checkout-action@v1
+
+ # - name: Install Rust Toolchain
+ # uses: ./.github/actions/rustup
+ # with:
+ # shared-key: 'benchmark_napi'
+ # save-cache: ${{ github.ref_name == 'main' }}
+
+ # - name: Install codspeed
+ # uses: taiki-e/install-action@v2
+ # with:
+ # tool: cargo-codspeed
+
+ # - name: Install Node.js
+ # uses: actions/setup-node@v4
+ # with:
+ # node-version: 20
+ # registry-url: 'https://registry.npmjs.org'
+
+ # - name: Start bench results interceptor server
+ # working-directory: ./tasks/benchmark/codspeed
+ # env:
+ # COMPONENT: 'parser_napi'
+ # run: |
+ # corepack enable
+ # pnpm install
+ # node capture.mjs &
+
+ # # CodSpeed gets measurements completely off for NAPI if run in `CodSpeedHQ/action`,
+ # # so instead run real benchmark without CodSpeed's instrumentation and save the results.
+ # # Then "Run Rust benchmark" step below runs a loop of some simple Rust code the number
+ # # of times required to take same amount of time as the real benchmark took.
+ # # This is all a workaround for https://github.com/CodSpeedHQ/action/issues/96
+ # - name: Build NAPI Benchmark
+ # working-directory: ./napi/parser
+ # run: |
+ # corepack enable
+ # pnpm install
+ # pnpm build
+
+ # - name: Run NAPI Benchmark
+ # working-directory: ./napi/parser
+ # run: node parse.bench.mjs
+
+ # - name: Build Rust benchmark
+ # env:
+ # RUSTFLAGS: "-C debuginfo=2 -C strip=none -g --cfg codspeed"
+ # shell: bash
+ # run: |
+ # cargo build --release -p oxc_benchmark --bench parser_napi --no-default-features --features codspeed_napi
+ # mkdir -p target/codspeed/oxc_benchmark/
+ # mv target/release/deps/parser_napi-* target/codspeed/oxc_benchmark
+ # rm -rf target/codspeed/oxc_benchmark/*.d
+
+ # - name: Run Rust benchmark
+ # uses: CodSpeedHQ/action@v2
+ # timeout-minutes: 30
+ # with:
+ # run: cargo codspeed run
+ # # Dummy token for tokenless runs, to suppress logging hash of metadata JSON (see `upload.mjs`)
+ # token: ${{ secrets.CODSPEED_TOKEN || 'dummy' }}
+ # upload-url: http://localhost:${{ env.INTERCEPT_PORT }}/upload
+
+ # - name: Upload bench data artefact
+ # uses: actions/upload-artifact@v4
+ # with:
+ # name: 'parser_napi'
+ # path: ${{ env.DATA_DIR }}
+ # if-no-files-found: error
+ # retention-days: 1
+
  upload:
  name: Upload benchmarks
  # needs: [benchmark, benchmark-napi]
@@ -203,8 +236,9 @@ jobs:
  - name: Download artefacts
  uses: actions/download-artifact@v4
  with:
- path: ${{ env.DATA_DIR }}
  merge-multiple: true
+ pattern: result-*
+ path: ${{ env.DATA_DIR }} # env.DEFAULT_PORT from create_temp_dir.mjs
 
  - name: Upload to Codspeed
  working-directory: ./tasks/benchmark/codspeed

diff --git a/tasks/benchmark/benches/linter.rs b/tasks/benchmark/benches/linter.rs
@@ -1,4 +1,4 @@
-use std::{env, path::PathBuf, rc::Rc};
+use std::{path::PathBuf, rc::Rc};
 
 use oxc_allocator::Allocator;
 use oxc_benchmark::{criterion_group, criterion_main, BenchmarkId, Criterion};
@@ -11,15 +11,7 @@ use oxc_tasks_common::TestFiles;
 fn bench_linter(criterion: &mut Criterion) {
  let mut group = criterion.benchmark_group("linter");
 
- // If `FIXTURE` env is set, only run the specified benchmark. This is used for sharding in CI.
- let test_files = if let Ok(fixture_index) = env::var("FIXTURE") {
- let fixture_index = fixture_index.parse::<usize>().unwrap();
- TestFiles::complicated_one(fixture_index)
- } else {
- TestFiles::complicated()
- };
-
- for file in test_files.files() {
+ for file in TestFiles::linter().files() {
  let source_type = SourceType::from_path(&file.file_name).unwrap();
  group.bench_with_input(
  BenchmarkId::from_parameter(&file.file_name),

diff --git a/tasks/common/src/test_file.rs b/tasks/common/src/test_file.rs
@@ -52,6 +52,11 @@ impl TestFiles {
  Self { files: vec![file] }
  }
 
+ pub fn linter() -> Self {
+ let files = Self::complicated_urls().into_iter().take(2).map(TestFile::new).collect();
+ Self { files }
+ }
+
  fn complicated_urls() -> [&'static str; 5] {
  [
  // TypeScript syntax (2.81MB)