dnrops
/
anchor
mirror of https://github.com/project-serum/anchor.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

bench: Add a script to sync benchmark results for all versions (#2477)

This commit is contained in:
acheron 2023-05-03 09:09:41 +02:00 committed by GitHub
parent a195106117
commit c1667120e6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 236 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
tests/bench/Anchor.toml
View File

@ -3,13 +3,14 @@ cluster = "localnet"
wallet = "~/.config/solana/id.json"
[programs.localnet]
bench = "Fg6PaFpoGXkYsidMpWTK6W2BeZ7FEfcYkg476zPFsLnS"
bench = "Bench11111111111111111111111111111111111111"
[workspace]
members = ["programs/bench"]
[scripts]
test = "yarn run ts-mocha -t 1000000 -p ./tsconfig.json -t 1000000 tests/**/*.ts"
update-bench = "yarn run ts-node scripts/update-bench.ts"
test = "yarn run ts-mocha -t 1000000 -p ./tsconfig.json tests/**/*.ts"
sync = "yarn run ts-node scripts/sync.ts"
sync-markdown = "yarn run ts-node scripts/sync-markdown.ts"
generate-ix = "yarn run ts-node scripts/generate-ix.ts"
bump-version = "yarn run ts-node scripts/bump-version.ts"

14
tests/bench/README.md
View File

@ -4,21 +4,25 @@ The bench program and its tests are used to measure the performance of Anchor pr
## How
Create a program -> Write tests that measure usage -> Compare the results -> Save the new result
The script will check whether there is a difference between the current result and the last saved result(in `bench.json`) at the end of the tests. If the difference between the results is greater than 1%, the new data will be saved in `bench.json` and Markdown files in [/bench](https://github.com/coral-xyz/anchor/tree/master/bench) will be updated accordingly.
We run the same tests that measure some metric for each Anchor version starting from `0.27.0`. If the difference between the results is greater than 1%, the new data will be saved in `bench.json` and Markdown files in [/bench](https://github.com/coral-xyz/anchor/tree/master/bench) will be updated accordingly.
## Scripts
| :memo: TL;DR |
| :----------------------------------------------------------------------------------------------------------------------------- |
| If you've made changes to programs or tests in this directory, run `anchor run sync`, otherwise run `anchor test --skip-lint`. |
`anchor test --skip-lint`: Run all tests and update benchmark files when necessary. This is the only command that needs to be run for most use cases.
---
The following scripts are useful when making changes to how benchmarking works.
`anchor run update-bench`: Update Markdown files in [/bench](https://github.com/coral-xyz/anchor/tree/master/bench) based on the data from `bench.json`.
`anchor run sync`: Sync all benchmark files by running tests for each version. If you've made changes to the bench program or its tests, you should run this command to sync the results.
`anchor run generate-ix`: Generate instructions with repetitive accounts.
`anchor run sync-markdown`: Sync Markdown files in [/bench](https://github.com/coral-xyz/anchor/tree/master/bench) based on the data from `bench.json`.
`anchor run generate-ix`: Generate program instructions with repetitive accounts.
---

5
tests/bench/programs/bench/Cargo.toml
View File

@ -14,3 +14,8 @@ cpi = ["no-entrypoint"]
[dependencies]
anchor-lang = { path = "../../../../lang" }
anchor-spl = { path = "../../../../spl" }
# TODO: Remove this and store lock files for each version instead.
# Latest solana version(1.14.17) as of 2023-05-01 comes with rustc 1.62.0-dev but MSRV for latest
# version of this crate is 1.64.0. See https://github.com/solana-labs/solana/pull/31418
winnow = "=0.4.1"

4
tests/bench/programs/bench/src/lib.rs
View File

@ -1,9 +1,11 @@
//! This program is used to measure the performance of Anchor programs.
//!
//! If you are making a change to this program, run `anchor run sync`.
use anchor_lang::prelude::*;
use anchor_spl::token_interface::{Mint, TokenAccount, TokenInterface};
declare_id!("Fg6PaFpoGXkYsidMpWTK6W2BeZ7FEfcYkg476zPFsLnS");
declare_id!("Bench11111111111111111111111111111111111111");
#[program]
pub mod bench {

20
tests/bench/scripts/update-bench.ts → tests/bench/scripts/sync-markdown.ts
View File

@ -1,4 +1,4 @@
/** Update Markdown files in /bench */
/** Sync Markdown files in /bench based on the data from bench.json */
import { BenchData, Markdown } from "./utils";
@ -33,20 +33,28 @@ import { BenchData, Markdown } from "./utils";
bench.compareComputeUnits(
newComputeUnitsResult,
oldComputeUnitsResult,
(ixName, newComputeUnits, oldComputeUnits) => {
({ ixName, newComputeUnits, oldComputeUnits }) => {
if (newComputeUnits === null) {
// Deleted instruction
return;
}
let changeText;
if (oldComputeUnits === null) {
// New instruction
changeText = "N/A";
} else {
const percentChange = (
(newComputeUnits / oldComputeUnits - 1) *
100
).toFixed(2);
let changeText;
if (isNaN(oldComputeUnits)) {
changeText = "N/A";
} else if (+percentChange > 0) {
if (+percentChange > 0) {
changeText = `🔴 **+${percentChange}%**`;
} else {
changeText = `🟢 **${percentChange}%**`;
}
}
table.insert(ixName, newComputeUnits.toString(), changeText);
},

66
tests/bench/scripts/sync.ts Normal file
View File

@ -0,0 +1,66 @@
/**
* Sync all saved data by re-running the tests for each version.
*
* This script should be used when the bench program or its tests has changed
* and all data needs to be updated.
*/
import path from "path";
import { spawnSync } from "child_process";
import { ANCHOR_VERSION_ARG, BenchData, Toml } from "./utils";
(async () => {
const bench = await BenchData.open();
const cargoToml = await Toml.open(
path.join("..", "programs", "bench", "Cargo.toml")
);
const anchorToml = await Toml.open(path.join("..", "Anchor.toml"));
for (const version of bench.getVersions()) {
console.log(`Updating '${version}'...`);
const isUnreleased = version === "unreleased";
// Update the anchor dependency versions
for (const dependency of ["lang", "spl"]) {
cargoToml.replaceValue(`anchor-${dependency}`, () => {
return isUnreleased
? `{ path = "../../../../${dependency}" }`
: `"${version}"`;
});
}
// Save Cargo.toml
await cargoToml.save();
// Update `anchor test` command to pass version in Anchor.toml
anchorToml.replaceValue(
"test",
(cmd) => {
return cmd.includes(ANCHOR_VERSION_ARG)
? cmd.replace(
new RegExp(`\\s*${ANCHOR_VERSION_ARG}\\s+(.+)`),
(arg, ver) => (isUnreleased ? "" : arg.replace(ver, version))
)
: `${cmd} ${ANCHOR_VERSION_ARG} ${version}`;
},
{ insideQuotes: true }
);
// Save Anchor.toml
await anchorToml.save();
// Run the command to update the current version's results
const result = spawnSync("anchor", ["test", "--skip-lint"]);
console.log(result.output.toString());
// Check for failure
if (result.status !== 0) {
console.error("Please fix the error and re-run this command.");
process.exitCode = 1;
return;
}
}
})();

140
tests/bench/scripts/utils.ts
View File

@ -2,6 +2,9 @@ import * as fs from "fs/promises";
import path from "path";
import { spawnSync } from "child_process";
/** Version that is used in bench data file */
export type Version = "unreleased" | (`${number}.${number}.${number}` & {});
/** Persistent benchmark data(mapping of `Version -> Data`) */
type Bench = {
[key: string]: {
@ -21,7 +24,10 @@ export type ComputeUnits = { [key: string]: number };
export const THRESHOLD_PERCENTAGE = 1;
/** Path to the benchmark Markdown files */
export const BENCH_DIR_PATH = "../../bench";
export const BENCH_DIR_PATH = path.join("..", "..", "bench");
/** Command line argument for Anchor version */
export const ANCHOR_VERSION_ARG = "--anchor-version";
/** Utility class to handle benchmark data related operations */
export class BenchData {
@ -56,43 +62,74 @@ export class BenchData {
}
/** Get the stored results based on version */
get(version: string) {
get(version: Version) {
return this.#data[version];
}
/** Get unreleased version results */
getUnreleased() {
return this.get("unreleased");
}
/** Get all versions */
getVersions() {
return Object.keys(this.#data);
return Object.keys(this.#data) as Version[];
}
/** Compare and update compute units changes */
compareComputeUnits(
newComputeUnitsResult: ComputeUnits,
oldComputeUnitsResult: ComputeUnits,
changeCb: (
ixName: string,
newComputeUnits: number,
oldComputeUnits: number
) => void,
changeCb: (args: {
ixName: string;
newComputeUnits: number | null;
oldComputeUnits: number | null;
}) => void,
noChangeCb?: (ixName: string, computeUnits: number) => void
) {
let needsUpdate = false;
const checkIxs = (
comparedFrom: ComputeUnits,
comparedTo: ComputeUnits,
cb: (ixName: string, computeUnits: number) => void
) => {
for (const ixName in comparedFrom) {
if (comparedTo[ixName] === undefined) {
cb(ixName, comparedFrom[ixName]);
}
}
};
// New instruction
checkIxs(
newComputeUnitsResult,
oldComputeUnitsResult,
(ixName, computeUnits) => {
console.log(`New instruction '${ixName}'`);
changeCb({
ixName,
newComputeUnits: computeUnits,
oldComputeUnits: null,
});
needsUpdate = true;
}
);
// Deleted instruction
checkIxs(
oldComputeUnitsResult,
newComputeUnitsResult,
(ixName, computeUnits) => {
console.log(`Deleted instruction '${ixName}'`);
changeCb({
ixName,
newComputeUnits: null,
oldComputeUnits: computeUnits,
});
needsUpdate = true;
}
);
// Compare compute units changes
for (const ixName in newComputeUnitsResult) {
const oldComputeUnits = oldComputeUnitsResult[ixName];
const newComputeUnits = newComputeUnitsResult[ixName];
if (!oldComputeUnits) {
console.log(`New instruction '${ixName}'`);
needsUpdate = true;
changeCb(ixName, newComputeUnits, NaN);
continue;
}
const percentage = THRESHOLD_PERCENTAGE / 100;
const oldMaximumAllowedDelta = oldComputeUnits * percentage;
@ -119,8 +156,12 @@ export class BenchData {
`Compute units change '${ixName}' (${oldComputeUnits} -> ${newComputeUnits})`
);
changeCb({
ixName,
newComputeUnits,
oldComputeUnits,
});
needsUpdate = true;
changeCb(ixName, newComputeUnits, oldComputeUnits);
} else {
noChangeCb?.(ixName, newComputeUnits);
}
@ -131,14 +172,14 @@ export class BenchData {
/** Bump benchmark data version to the given version */
bumpVersion(newVersion: string) {
const versions = Object.keys(this.#data);
const unreleasedVersion = versions[versions.length - 1];
if (this.#data[newVersion]) {
console.error(`Version '${newVersion}' already exists!`);
process.exit(1);
}
const versions = this.getVersions();
const unreleasedVersion = versions[versions.length - 1];
// Add the new version
this.#data[newVersion] = this.get(unreleasedVersion);
@ -296,3 +337,56 @@ class MarkdownTable {
);
}
}
/** Utility class to handle TOML related operations */
export class Toml {
/** TOML filepath */
#path: string;
/** TOML text */
#text: string;
constructor(path: string, text: string) {
this.#path = path;
this.#text = text;
}
/** Open the TOML file */
static async open(tomlPath: string) {
tomlPath = path.join(__dirname, tomlPath);
const text = await fs.readFile(tomlPath, {
encoding: "utf8",
});
return new Toml(tomlPath, text);
}
/** Save the TOML file */
async save() {
await fs.writeFile(this.#path, this.#text);
}
/** Replace the value for the given key */
replaceValue(
key: string,
cb: (previous: string) => string,
opts?: { insideQuotes: boolean }
) {
this.#text = this.#text.replace(
new RegExp(`${key}\\s*=\\s*${opts?.insideQuotes ? `"(.*)"` : "(.*)"}`),
(line, value) => line.replace(value, cb(value))
);
}
}
/**
* Get Anchor version from the passed arguments.
*
* Defaults to `unreleased`.
*/
export const getVersionFromArgs = () => {
const args = process.argv;
const anchorVersionArgIndex = args.indexOf(ANCHOR_VERSION_ARG);
return anchorVersionArgIndex === -1
? "unreleased"
: (args[anchorVersionArgIndex + 1] as Version);
};

20
tests/bench/tests/compute-units.ts
View File

@ -3,7 +3,7 @@ import * as token from "@coral-xyz/spl-token";
import { spawnSync } from "child_process";
import { Bench, IDL } from "../target/types/bench";
import { BenchData, ComputeUnits } from "../scripts/utils";
import { BenchData, ComputeUnits, getVersionFromArgs } from "../scripts/utils";
describe(IDL.name, () => {
// Configure the client to use the local cluster
@ -226,12 +226,17 @@ describe(IDL.name, () => {
const bench = await BenchData.open();
// Compare and update compute units changes
const oldComputeUnits = bench.getUnreleased().computeUnits;
const version = getVersionFromArgs();
const oldComputeUnits = bench.get(version).computeUnits;
const { needsUpdate } = bench.compareComputeUnits(
computeUnits,
oldComputeUnits,
(ixName, newComputeUnits) => {
oldComputeUnits[ixName] = newComputeUnits;
({ ixName, newComputeUnits: newValue }) => {
if (newValue === null) {
delete oldComputeUnits[ixName];
} else {
oldComputeUnits[ixName] = newValue;
}
}
);
@ -239,10 +244,13 @@ describe(IDL.name, () => {
console.log("Updating benchmark files...");
// Save bench data file
// (needs to happen before running the `update-bench` script)
// (needs to happen before running the `sync-markdown` script)
await bench.save();
spawnSync("anchor", ["run", "update-bench"]);
// Only update markdown files on `unreleased` version
if (version === "unreleased") {
spawnSync("anchor", ["run", "sync-markdown"]);
}
}
});
});