name: Benchmark
on:
workflow_dispatch:
inputs:
repo:
type: string
description: GitHub repository to fetch from (default to the current repo)
pr_id:
type: number
required: true
description: The PR to test
commit:
required: true
type: string
description: The expect HEAD of the PR
category:
required: true
type: string
description: The category (or categories) of tests to run, for example buffers, cluster etc. Maps to a folders in node/benchmark
filter:
type: string
description: A substring to restrict the benchmarks to run in a category. e.g. `net-c2c`
runs:
type: number
default: 30
description: How many times to repeat each benchmark
post-comment:
type: boolean
description: Post a comment linking to the run, with the aggregated result once known.
token:
type: string
description: A GitHub token to post comments cross repository (not recommended)
permissions:
contents: read
jobs:
post-comment:
if: inputs.post-comment
outputs:
body: ${{ steps.comment.outputs.COMMENT_BODY }}
url: ${{ steps.comment.outputs.COMMENT_URL }}
runs-on: ubuntu-slim
permissions:
pull-requests: write
steps:
- name: Mark token input as sensitive
if: inputs.token != ''
run: printf '::add-mask::%s\n' "$COMMENT_TOKEN"
env:
COMMENT_TOKEN: ${{ inputs.token }}
- name: Add link to the current run
id: comment
run: |
FILTER_IF_SET=
[ -z "$FILTER" ] || FILTER_IF_SET=" / $FILTER"
COMMENT_BODY="Benchmark GHA (${CATEGORIES}${FILTER_IF_SET}): ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
echo "COMMENT_BODY=$COMMENT_BODY" >> "$GITHUB_OUTPUT"
echo "COMMENT_URL=$(gh pr comment -R "${REPO:-$GITHUB_REPOSITORY}" "$PR_ID" --body "$COMMENT_BODY")" >> "$GITHUB_OUTPUT"
env:
CATEGORIES: ${{ inputs.category }}
FILTER: ${{ inputs.filter }}
GH_TOKEN: ${{ inputs.token || github.token }}
REPO: ${{ inputs.repo }}
PR_ID: ${{ inputs.pr_id }}
build:
strategy:
fail-fast: true
matrix:
include:
- runner: ubuntu-24.04
system: x86_64-linux
perfetto: true
- runner: ubuntu-24.04-arm
system: aarch64-linux
- runner: macos-15-intel
system: x86_64-darwin
- runner: macos-latest
system: aarch64-darwin
name: '${{ matrix.system }}: with shared libraries${{ matrix.perfetto && '' and perfetto'' || '''' }}'
runs-on: ${{ matrix.runner }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.repo || github.repository }}
ref: refs/pull/${{ inputs.pr_id }}/merge
persist-credentials: false
fetch-depth: 2
- name: Validate PR head and roll back to base commit
run: |
[ "$(git rev-parse HEAD^2)" = "$EXPECTED_SHA" ]
git reset HEAD^ --hard
env:
EXPECTED_SHA: ${{ inputs.commit }}
- uses: cachix/install-nix-action@75c75c31ebece974e48ebc36b4ad125bd97bd97b # v31.11.3
with:
extra_nix_config: sandbox = true
- uses: cachix/cachix-action@38b082610b782e7e93e209c35fd730d399dee866 # v17
with:
# We do not pass any `authToken` to avoid polluting the cache with potentially untrusted code.
name: nodejs
- name: Configure sccache
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
core.exportVariable('SCCACHE_GHA_ENABLED', 'on');
core.exportVariable('SCCACHE_GHA_RW_MODE', 'READ_ONLY');
core.exportVariable('ACTIONS_CACHE_SERVICE_V2', 'on');
core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
- name: Build Node.js on the base commit
run: |
nix-shell \
-I nixpkgs=./tools/nix/pkgs.nix \
--pure --keep TAR_DIR --keep FLAKY_TESTS \
--keep SCCACHE_GHA_ENABLED --keep SCCACHE_GHA_RW_MODE \
--keep ACTIONS_CACHE_SERVICE_V2 --keep ACTIONS_RESULTS_URL --keep ACTIONS_RUNTIME_TOKEN \
--arg withPerfetto ${{ matrix.perfetto || false }} \
--arg ccache '(import <nixpkgs> {}).sccache' \
--run '
make build-ci -j4 V=1 && sccache --stop-server
' tools/nix/ci-shell.nix
mv out/Release/node base_node
- name: Checkout the merge commit
run: git reset FETCH_HEAD --hard
- name: Re-build Node.js on the merge commit
# Keep the compiler command unchanged for incremental builds, but never cache PR build outputs.
run: |
nix-shell \
-I nixpkgs=./tools/nix/pkgs.nix \
--pure --keep SCCACHE_LOCAL_RW_MODE \
--arg withPerfetto ${{ matrix.perfetto || false }} \
--arg ccache '(import <nixpkgs> {}).sccache' \
--run '
make build-ci -j4 V=1 ${{ contains(format(' {0} ', inputs.category), ' napi ') && '&& make bench-addons-build' || '' }}
' tools/nix/ci-shell.nix
env:
SCCACHE_LOCAL_RW_MODE: READ_ONLY
- name: Run benchmark
run: |
nix-shell \
-I nixpkgs=./tools/nix/pkgs.nix \
--pure --keep CATEGORIES --keep FILTER --keep RUNS --keep LC_ALL --keep LANG \
--arg loadJSBuiltinsDynamically false \
--arg ccache 'null' \
--arg icu 'null' \
--arg sharedLibDeps '{}' \
--arg devTools '[]' \
--run '
set -o pipefail
read -r -a categories <<< "$CATEGORIES"
./base_node benchmark/compare.js \
--filter "$FILTER" \
--runs "$RUNS" \
--old ./base_node --new ./node \
-- "${categories[@]}" \
| tee /dev/stderr \
> ${{ matrix.system }}.csv
echo "> [!WARNING] "
echo "> Do not take GHA benchmark results as face value, always confirm them"
echo "> using a dedicated machine, e.g. Jenkins CI."
echo
echo "Benchmark results:"
echo
echo '"'"'```'"'"'
Rscript benchmark/compare.R < ${{ matrix.system }}.csv
echo '"'"'```'"'"'
echo
echo "> [!WARNING] "
echo "> Do not take GHA benchmark results as face value, always confirm them"
echo "> using a dedicated machine, e.g. Jenkins CI."
' | tee /dev/stderr >> "$GITHUB_STEP_SUMMARY"
env:
CATEGORIES: ${{ inputs.category }}
FILTER: ${{ inputs.filter }}
RUNS: ${{ inputs.runs }}
- name: Upload raw benchmark results
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: csv-${{ matrix.system }}
path: ${{ matrix.system }}.csv
aggregate-results:
needs: [build, post-comment]
if: ${{ always() && needs.build.result == 'success' && needs.post-comment.result == (inputs.post-comment && 'success' || 'skipped') }}
name: Aggregate benchmark results
runs-on: ubuntu-slim
permissions:
contents: read
pull-requests: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
sparse-checkout: |
benchmark/*.R
tools/nix/*.nix
*.nix
sparse-checkout-cone-mode: false
- name: Download benchmark raw results
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
pattern: csv-*
merge-multiple: true
path: raw-results
- uses: cachix/install-nix-action@75c75c31ebece974e48ebc36b4ad125bd97bd97b # v31.11.3
with:
extra_nix_config: sandbox = true
- name: Benchmark results
run: |
nix-shell \
-I nixpkgs=./tools/nix/pkgs.nix \
--pure \
-E '(import <nixpkgs> {}).mkShellNoCC { buildInputs = import ./tools/nix/benchmarkTools.nix { withHttpBenchmarkDeps = false; }; }' \
--run '
export LC_ALL=C.UTF-8
echo "> [!WARNING] "
echo "> Do not take GHA benchmark results as face value, always confirm them"
echo "> using a dedicated machine, e.g. Jenkins CI."
echo
echo "Benchmark results:"
echo
echo '"'"'```'"'"'
awk "FNR==1 && NR!=1{next;}{print}" raw-results/*.csv | Rscript benchmark/compare.R
echo '"'"'```'"'"'
echo
echo "> [!WARNING] "
echo "> Do not take GHA benchmark results as face value, always confirm them"
echo "> using a dedicated machine, e.g. Jenkins CI."
' | tee /dev/stderr ${{ inputs.post-comment && 'body.txt' || '' }} >> "$GITHUB_STEP_SUMMARY"
- name: Mark token input as sensitive
if: inputs.token != ''
run: printf '::add-mask::%s\n' "$COMMENT_TOKEN"
env:
COMMENT_TOKEN: ${{ inputs.token }}
- name: Edit comment
if: inputs.post-comment
run: |
{
echo "$COMMENT_BODY"
echo
echo '<details><summary>Results</summary>'
echo
cat body.txt
echo
echo '</details>'
} | jq -Rrcs '{ body: . }' | gh api \
--method PATCH \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2026-03-10" \
"/repos/${REPO:-$GITHUB_REPOSITORY}/issues/comments/${COMMENT_URL##*-}" \
--input -
env:
GH_TOKEN: ${{ inputs.token || github.token }}
REPO: ${{ inputs.repo }}
COMMENT_BODY: ${{ needs.post-comment.outputs.body }}
COMMENT_URL: ${{ needs.post-comment.outputs.url }}
PR_ID: ${{ inputs.pr_id }}