diff --git a/tests/bench/Anchor.toml b/tests/bench/Anchor.toml index 8e0cf1d52..661873a26 100644 --- a/tests/bench/Anchor.toml +++ b/tests/bench/Anchor.toml @@ -3,13 +3,14 @@ cluster = "localnet" wallet = "~/.config/solana/id.json" [programs.localnet] -bench = "Fg6PaFpoGXkYsidMpWTK6W2BeZ7FEfcYkg476zPFsLnS" +bench = "Bench11111111111111111111111111111111111111" [workspace] members = ["programs/bench"] [scripts] -test = "yarn run ts-mocha -t 1000000 -p ./tsconfig.json -t 1000000 tests/**/*.ts" -update-bench = "yarn run ts-node scripts/update-bench.ts" +test = "yarn run ts-mocha -t 1000000 -p ./tsconfig.json tests/**/*.ts" +sync = "yarn run ts-node scripts/sync.ts" +sync-markdown = "yarn run ts-node scripts/sync-markdown.ts" generate-ix = "yarn run ts-node scripts/generate-ix.ts" bump-version = "yarn run ts-node scripts/bump-version.ts" diff --git a/tests/bench/README.md b/tests/bench/README.md index b57b50d0f..061e1d54a 100644 --- a/tests/bench/README.md +++ b/tests/bench/README.md @@ -4,21 +4,25 @@ The bench program and its tests are used to measure the performance of Anchor pr ## How -Create a program -> Write tests that measure usage -> Compare the results -> Save the new result - -The script will check whether there is a difference between the current result and the last saved result(in `bench.json`) at the end of the tests. If the difference between the results is greater than 1%, the new data will be saved in `bench.json` and Markdown files in [/bench](https://github.com/coral-xyz/anchor/tree/master/bench) will be updated accordingly. +We run the same tests that measure some metric for each Anchor version starting from `0.27.0`. If the difference between the results is greater than 1%, the new data will be saved in `bench.json` and Markdown files in [/bench](https://github.com/coral-xyz/anchor/tree/master/bench) will be updated accordingly. ## Scripts +| :memo: TL;DR | +| :----------------------------------------------------------------------------------------------------------------------------- | +| If you've made changes to programs or tests in this directory, run `anchor run sync`, otherwise run `anchor test --skip-lint`. | + `anchor test --skip-lint`: Run all tests and update benchmark files when necessary. This is the only command that needs to be run for most use cases. --- The following scripts are useful when making changes to how benchmarking works. -`anchor run update-bench`: Update Markdown files in [/bench](https://github.com/coral-xyz/anchor/tree/master/bench) based on the data from `bench.json`. +`anchor run sync`: Sync all benchmark files by running tests for each version. If you've made changes to the bench program or its tests, you should run this command to sync the results. -`anchor run generate-ix`: Generate instructions with repetitive accounts. +`anchor run sync-markdown`: Sync Markdown files in [/bench](https://github.com/coral-xyz/anchor/tree/master/bench) based on the data from `bench.json`. + +`anchor run generate-ix`: Generate program instructions with repetitive accounts. --- diff --git a/tests/bench/programs/bench/Cargo.toml b/tests/bench/programs/bench/Cargo.toml index d7b162620..1d21010ac 100644 --- a/tests/bench/programs/bench/Cargo.toml +++ b/tests/bench/programs/bench/Cargo.toml @@ -14,3 +14,8 @@ cpi = ["no-entrypoint"] [dependencies] anchor-lang = { path = "../../../../lang" } anchor-spl = { path = "../../../../spl" } + +# TODO: Remove this and store lock files for each version instead. +# Latest solana version(1.14.17) as of 2023-05-01 comes with rustc 1.62.0-dev but MSRV for latest +# version of this crate is 1.64.0. See https://github.com/solana-labs/solana/pull/31418 +winnow = "=0.4.1" diff --git a/tests/bench/programs/bench/src/lib.rs b/tests/bench/programs/bench/src/lib.rs index e9a2a6b8e..acfa1bd5d 100644 --- a/tests/bench/programs/bench/src/lib.rs +++ b/tests/bench/programs/bench/src/lib.rs @@ -1,9 +1,11 @@ //! This program is used to measure the performance of Anchor programs. +//! +//! If you are making a change to this program, run `anchor run sync`. use anchor_lang::prelude::*; use anchor_spl::token_interface::{Mint, TokenAccount, TokenInterface}; -declare_id!("Fg6PaFpoGXkYsidMpWTK6W2BeZ7FEfcYkg476zPFsLnS"); +declare_id!("Bench11111111111111111111111111111111111111"); #[program] pub mod bench { diff --git a/tests/bench/scripts/update-bench.ts b/tests/bench/scripts/sync-markdown.ts similarity index 68% rename from tests/bench/scripts/update-bench.ts rename to tests/bench/scripts/sync-markdown.ts index 9b6cd063b..1c8e1eab3 100644 --- a/tests/bench/scripts/update-bench.ts +++ b/tests/bench/scripts/sync-markdown.ts @@ -1,4 +1,4 @@ -/** Update Markdown files in /bench */ +/** Sync Markdown files in /bench based on the data from bench.json */ import { BenchData, Markdown } from "./utils"; @@ -33,19 +33,27 @@ import { BenchData, Markdown } from "./utils"; bench.compareComputeUnits( newComputeUnitsResult, oldComputeUnitsResult, - (ixName, newComputeUnits, oldComputeUnits) => { - const percentChange = ( - (newComputeUnits / oldComputeUnits - 1) * - 100 - ).toFixed(2); + ({ ixName, newComputeUnits, oldComputeUnits }) => { + if (newComputeUnits === null) { + // Deleted instruction + return; + } let changeText; - if (isNaN(oldComputeUnits)) { + if (oldComputeUnits === null) { + // New instruction changeText = "N/A"; - } else if (+percentChange > 0) { - changeText = `🔴 **+${percentChange}%**`; } else { - changeText = `🟢 **${percentChange}%**`; + const percentChange = ( + (newComputeUnits / oldComputeUnits - 1) * + 100 + ).toFixed(2); + + if (+percentChange > 0) { + changeText = `🔴 **+${percentChange}%**`; + } else { + changeText = `🟢 **${percentChange}%**`; + } } table.insert(ixName, newComputeUnits.toString(), changeText); diff --git a/tests/bench/scripts/sync.ts b/tests/bench/scripts/sync.ts new file mode 100644 index 000000000..b535921d1 --- /dev/null +++ b/tests/bench/scripts/sync.ts @@ -0,0 +1,66 @@ +/** + * Sync all saved data by re-running the tests for each version. + * + * This script should be used when the bench program or its tests has changed + * and all data needs to be updated. + */ + +import path from "path"; +import { spawnSync } from "child_process"; + +import { ANCHOR_VERSION_ARG, BenchData, Toml } from "./utils"; + +(async () => { + const bench = await BenchData.open(); + + const cargoToml = await Toml.open( + path.join("..", "programs", "bench", "Cargo.toml") + ); + const anchorToml = await Toml.open(path.join("..", "Anchor.toml")); + + for (const version of bench.getVersions()) { + console.log(`Updating '${version}'...`); + + const isUnreleased = version === "unreleased"; + + // Update the anchor dependency versions + for (const dependency of ["lang", "spl"]) { + cargoToml.replaceValue(`anchor-${dependency}`, () => { + return isUnreleased + ? `{ path = "../../../../${dependency}" }` + : `"${version}"`; + }); + } + + // Save Cargo.toml + await cargoToml.save(); + + // Update `anchor test` command to pass version in Anchor.toml + anchorToml.replaceValue( + "test", + (cmd) => { + return cmd.includes(ANCHOR_VERSION_ARG) + ? cmd.replace( + new RegExp(`\\s*${ANCHOR_VERSION_ARG}\\s+(.+)`), + (arg, ver) => (isUnreleased ? "" : arg.replace(ver, version)) + ) + : `${cmd} ${ANCHOR_VERSION_ARG} ${version}`; + }, + { insideQuotes: true } + ); + + // Save Anchor.toml + await anchorToml.save(); + + // Run the command to update the current version's results + const result = spawnSync("anchor", ["test", "--skip-lint"]); + console.log(result.output.toString()); + + // Check for failure + if (result.status !== 0) { + console.error("Please fix the error and re-run this command."); + process.exitCode = 1; + return; + } + } +})(); diff --git a/tests/bench/scripts/utils.ts b/tests/bench/scripts/utils.ts index 8350cbbb6..02bd38804 100644 --- a/tests/bench/scripts/utils.ts +++ b/tests/bench/scripts/utils.ts @@ -2,6 +2,9 @@ import * as fs from "fs/promises"; import path from "path"; import { spawnSync } from "child_process"; +/** Version that is used in bench data file */ +export type Version = "unreleased" | (`${number}.${number}.${number}` & {}); + /** Persistent benchmark data(mapping of `Version -> Data`) */ type Bench = { [key: string]: { @@ -21,7 +24,10 @@ export type ComputeUnits = { [key: string]: number }; export const THRESHOLD_PERCENTAGE = 1; /** Path to the benchmark Markdown files */ -export const BENCH_DIR_PATH = "../../bench"; +export const BENCH_DIR_PATH = path.join("..", "..", "bench"); + +/** Command line argument for Anchor version */ +export const ANCHOR_VERSION_ARG = "--anchor-version"; /** Utility class to handle benchmark data related operations */ export class BenchData { @@ -56,43 +62,74 @@ export class BenchData { } /** Get the stored results based on version */ - get(version: string) { + get(version: Version) { return this.#data[version]; } - /** Get unreleased version results */ - getUnreleased() { - return this.get("unreleased"); - } - /** Get all versions */ getVersions() { - return Object.keys(this.#data); + return Object.keys(this.#data) as Version[]; } /** Compare and update compute units changes */ compareComputeUnits( newComputeUnitsResult: ComputeUnits, oldComputeUnitsResult: ComputeUnits, - changeCb: ( - ixName: string, - newComputeUnits: number, - oldComputeUnits: number - ) => void, + changeCb: (args: { + ixName: string; + newComputeUnits: number | null; + oldComputeUnits: number | null; + }) => void, noChangeCb?: (ixName: string, computeUnits: number) => void ) { let needsUpdate = false; + const checkIxs = ( + comparedFrom: ComputeUnits, + comparedTo: ComputeUnits, + cb: (ixName: string, computeUnits: number) => void + ) => { + for (const ixName in comparedFrom) { + if (comparedTo[ixName] === undefined) { + cb(ixName, comparedFrom[ixName]); + } + } + }; + + // New instruction + checkIxs( + newComputeUnitsResult, + oldComputeUnitsResult, + (ixName, computeUnits) => { + console.log(`New instruction '${ixName}'`); + changeCb({ + ixName, + newComputeUnits: computeUnits, + oldComputeUnits: null, + }); + needsUpdate = true; + } + ); + + // Deleted instruction + checkIxs( + oldComputeUnitsResult, + newComputeUnitsResult, + (ixName, computeUnits) => { + console.log(`Deleted instruction '${ixName}'`); + changeCb({ + ixName, + newComputeUnits: null, + oldComputeUnits: computeUnits, + }); + needsUpdate = true; + } + ); + // Compare compute units changes for (const ixName in newComputeUnitsResult) { const oldComputeUnits = oldComputeUnitsResult[ixName]; const newComputeUnits = newComputeUnitsResult[ixName]; - if (!oldComputeUnits) { - console.log(`New instruction '${ixName}'`); - needsUpdate = true; - changeCb(ixName, newComputeUnits, NaN); - continue; - } const percentage = THRESHOLD_PERCENTAGE / 100; const oldMaximumAllowedDelta = oldComputeUnits * percentage; @@ -119,8 +156,12 @@ export class BenchData { `Compute units change '${ixName}' (${oldComputeUnits} -> ${newComputeUnits})` ); + changeCb({ + ixName, + newComputeUnits, + oldComputeUnits, + }); needsUpdate = true; - changeCb(ixName, newComputeUnits, oldComputeUnits); } else { noChangeCb?.(ixName, newComputeUnits); } @@ -131,14 +172,14 @@ export class BenchData { /** Bump benchmark data version to the given version */ bumpVersion(newVersion: string) { - const versions = Object.keys(this.#data); - const unreleasedVersion = versions[versions.length - 1]; - if (this.#data[newVersion]) { console.error(`Version '${newVersion}' already exists!`); process.exit(1); } + const versions = this.getVersions(); + const unreleasedVersion = versions[versions.length - 1]; + // Add the new version this.#data[newVersion] = this.get(unreleasedVersion); @@ -296,3 +337,56 @@ class MarkdownTable { ); } } + +/** Utility class to handle TOML related operations */ +export class Toml { + /** TOML filepath */ + #path: string; + + /** TOML text */ + #text: string; + + constructor(path: string, text: string) { + this.#path = path; + this.#text = text; + } + + /** Open the TOML file */ + static async open(tomlPath: string) { + tomlPath = path.join(__dirname, tomlPath); + const text = await fs.readFile(tomlPath, { + encoding: "utf8", + }); + return new Toml(tomlPath, text); + } + + /** Save the TOML file */ + async save() { + await fs.writeFile(this.#path, this.#text); + } + + /** Replace the value for the given key */ + replaceValue( + key: string, + cb: (previous: string) => string, + opts?: { insideQuotes: boolean } + ) { + this.#text = this.#text.replace( + new RegExp(`${key}\\s*=\\s*${opts?.insideQuotes ? `"(.*)"` : "(.*)"}`), + (line, value) => line.replace(value, cb(value)) + ); + } +} + +/** + * Get Anchor version from the passed arguments. + * + * Defaults to `unreleased`. + */ +export const getVersionFromArgs = () => { + const args = process.argv; + const anchorVersionArgIndex = args.indexOf(ANCHOR_VERSION_ARG); + return anchorVersionArgIndex === -1 + ? "unreleased" + : (args[anchorVersionArgIndex + 1] as Version); +}; diff --git a/tests/bench/tests/compute-units.ts b/tests/bench/tests/compute-units.ts index 4b5c8fbc2..77e0a778d 100644 --- a/tests/bench/tests/compute-units.ts +++ b/tests/bench/tests/compute-units.ts @@ -3,7 +3,7 @@ import * as token from "@coral-xyz/spl-token"; import { spawnSync } from "child_process"; import { Bench, IDL } from "../target/types/bench"; -import { BenchData, ComputeUnits } from "../scripts/utils"; +import { BenchData, ComputeUnits, getVersionFromArgs } from "../scripts/utils"; describe(IDL.name, () => { // Configure the client to use the local cluster @@ -226,12 +226,17 @@ describe(IDL.name, () => { const bench = await BenchData.open(); // Compare and update compute units changes - const oldComputeUnits = bench.getUnreleased().computeUnits; + const version = getVersionFromArgs(); + const oldComputeUnits = bench.get(version).computeUnits; const { needsUpdate } = bench.compareComputeUnits( computeUnits, oldComputeUnits, - (ixName, newComputeUnits) => { - oldComputeUnits[ixName] = newComputeUnits; + ({ ixName, newComputeUnits: newValue }) => { + if (newValue === null) { + delete oldComputeUnits[ixName]; + } else { + oldComputeUnits[ixName] = newValue; + } } ); @@ -239,10 +244,13 @@ describe(IDL.name, () => { console.log("Updating benchmark files..."); // Save bench data file - // (needs to happen before running the `update-bench` script) + // (needs to happen before running the `sync-markdown` script) await bench.save(); - spawnSync("anchor", ["run", "update-bench"]); + // Only update markdown files on `unreleased` version + if (version === "unreleased") { + spawnSync("anchor", ["run", "sync-markdown"]); + } } }); });