Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve benchmark reporting #16

Merged
merged 1 commit into from
Nov 6, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 102 additions & 0 deletions .github/workflows/benchmarks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
name: benchmarks

on: [pull_request]

jobs:
size:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 50
- uses: actions/setup-node@v4
with:
node-version: '20.x'
- name: 'Setup temporary files'
run: |
echo "BASE_JSON=$(mktemp)" >> $GITHUB_ENV
echo "PATCH_JSON=$(mktemp)" >> $GITHUB_ENV
- name: 'Benchmark base'
run: |
git checkout -f ${{ github.event.pull_request.base.sha }}
npm install --loglevel error
if npm run benchmark:size -- -o ${{ env.BASE_JSON }}; then
echo "Ran successfully on base branch"
else
echo "{}" > ${{ env.BASE_JSON }} # Empty JSON as default
echo "Benchmark script not found on base branch, using default values"
fi
- name: 'Benchmark patch'
run: |
git checkout -f ${{ github.event.pull_request.head.sha }}
npm install --loglevel error
npm run benchmark:size -- -o ${{ env.PATCH_JSON }}
echo "Ran successfully on patch branch"
- name: 'Collect results'
id: collect
run: |
echo "table<<EOF" >> $GITHUB_OUTPUT
npm run benchmark:compare -- ${{ env.BASE_JSON }} ${{ env.PATCH_JSON }} >> markdown
cat markdown >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
- name: 'Post comment'
uses: edumserrano/find-create-or-update-comment@v3
with:
issue-number: ${{ github.event.pull_request.number }}
body-includes: '<!-- workflow-benchmarks-size-data -->'
comment-author: 'github-actions[bot]'
body: |
<!-- workflow-benchmarks-size-data -->
### workflow: benchmarks/size
Comparison of minified (terser) and compressed (brotli) size results, measured in bytes. Smaller is better.
${{ steps.collect.outputs.table }}
edit-mode: replace

perf:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 50
- uses: actions/setup-node@v4
with:
node-version: '20.x'
- name: 'Setup temporary files'
run: |
echo "BASE_JSON=$(mktemp)" >> $GITHUB_ENV
echo "PATCH_JSON=$(mktemp)" >> $GITHUB_ENV
- name: 'Benchmark base'
run: |
git checkout -f ${{ github.event.pull_request.base.sha }}
npm install --loglevel error
if npm run benchmark:perf -- -o ${{ env.BASE_JSON }}; then
echo "Ran successfully on base branch"
else
echo "{}" > ${{ env.BASE_JSON }} # Empty JSON as default
echo "Benchmark script not found on base branch, using default values"
fi
- name: 'Benchmark patch'
run: |
git checkout -f ${{ github.event.pull_request.head.sha }}
npm install --loglevel error
npm run benchmark:perf -- -o ${{ env.PATCH_JSON }}
echo "Ran successfully on patch branch"
- name: 'Collect results'
id: collect
run: |
echo "table<<EOF" >> $GITHUB_OUTPUT
npm run benchmark:compare -- ${{ env.BASE_JSON }} ${{ env.PATCH_JSON }} >> markdown
cat markdown >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
- name: 'Post comment'
uses: edumserrano/find-create-or-update-comment@v3
with:
issue-number: ${{ github.event.pull_request.number }}
body-includes: '<!-- workflow-benchmarks-perf-data -->'
comment-author: 'github-actions[bot]'
body: |
<!-- workflow-benchmarks-perf-data -->
### workflow: benchmarks/perf
Comparison of performance test results, measured in operations per second. Larger is better.
${{ steps.collect.outputs.table }}
edit-mode: replace
64 changes: 0 additions & 64 deletions .github/workflows/performance.yml

This file was deleted.

2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
*.log*
coverage
dist
logs
node_modules
103 changes: 103 additions & 0 deletions benchmark/compare.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/

'use strict';

const fs = require('fs');

function readJsonFile(filePath) {
try {
const fileContents = fs.readFileSync(filePath, 'utf8');
const data = JSON.parse(fileContents);
return data;
} catch (error) {
console.error(`Error reading file ${filePath}:`, error);
return null;
}
}

function mergeData(base, patch) {
const merged = {};
function addToMerged(data, fileIndex) {
Object.keys(data).forEach((key) => {
if (merged[key] == null) {
merged[key] = {};
}
Object.keys(data[key]).forEach((subKey) => {
if (merged[key][subKey] == null) {
merged[key][subKey] = {};
}
merged[key][subKey][fileIndex] = data[key][subKey];
});
});
}
if (base != null) {
addToMerged(base, 1);
}
if (patch != null) {
addToMerged(patch, 2);
}
return merged;
}

function generateComparisonData(results) {
const baseResult = parseInt(results[1], 10);
const patchResult = parseInt(results[2], 10);
const isValidBase = !isNaN(baseResult);
const isValidPatch = !isNaN(patchResult);
let icon = '',
ratioFixed = '';

if (isValidBase && isValidPatch) {
const ratio = patchResult / baseResult;
ratioFixed = ratio.toFixed(2);
if (ratio < 0.95 || ratio > 1.05) {
icon = '**!!**';
} else if (ratio < 1) {
icon = '-';
} else if (ratio > 1) {
icon = '+';
}
}

return {
baseResult: isValidBase ? baseResult.toLocaleString() : '',
patchResult: isValidPatch ? patchResult.toLocaleString() : '',
ratio: ratioFixed,
icon,
};
}

function generateMarkdownTable(mergedData) {
const rows = [];
rows.push('| **Results** | **Base** | **Patch** | **Ratio** | |');
rows.push('| :--- | ---: | ---: | ---: | ---: |');
Object.keys(mergedData).forEach((suiteName) => {
rows.push('| | | | |');
rows.push(`| **${suiteName}** | | | | |`);
Object.keys(mergedData[suiteName]).forEach((test) => {
const results = mergedData[suiteName][test];
const { baseResult, patchResult, ratio, icon } =
generateComparisonData(results);
rows.push(
`| &middot; ${test} | ${baseResult} | ${patchResult} | ${ratio} | ${icon} |`
);
});
});
return rows.join('\n');
}

/**
* Compare up to 2 different benchmark runs
*/
const args = process.argv.slice(2);
const baseResults = args[0] ? readJsonFile(args[0]) : null;
const patchResults = args[1] ? readJsonFile(args[1]) : null;
const mergedData = mergeData(baseResults, patchResults);
const markdownTable = generateMarkdownTable(mergedData);

console.log(markdownTable);
Loading