cli: run a subset of the test suites (#1864)

* cli: run subset of tests with --run

* changelog

* test

* fix run test path

* test: added missing package.json

* cli: run subset of tests with --run

* changelog

* test

* fix run test path

* test: added missing package.json

Co-authored-by: Armani Ferrante <armaniferrante@gmail.com>
Co-authored-by: henrye <henry@notanemail>

.github/workflows/tests.yaml

@@ -401,6 +401,8 @@ jobs:
             path: tests/cpi-returns
           - cmd: cd tests/multiple-suites && anchor test --skip-lint && npx tsc --noEmit
             path: tests/multiple-suites
+          - cmd: cd tests/multiple-suites-run-single && anchor test --skip-lint --run tests/should-run && npx tsc --noEmit
+            path: tests/multiple-suites-run-single
           - cmd: cd tests/pda-derivation && anchor test --skip-lint && npx tsc --noEmit
             path: tests/pda-derivation
           - cmd: cd tests/relations-derivation && anchor test --skip-lint && npx tsc --noEmit

CHANGELOG.md

@@ -12,6 +12,7 @@ The minor version will be incremented upon a breaking change and the patch versi
 
 ### Features
 
+- cli: Add `--run` to `anchor test` for running a subset of test suites ([#1828](https://github.com/project-serum/anchor/issues/1828)).
 - client: Add `transaction` functions to RequestBuilder ([#1958](https://github.com/coral-xyz/anchor/pull/1958)).
 - spl: Add `create_metadata_accounts_v3` and `set_collection_size` wrappers ([#2119](https://github.com/coral-xyz/anchor/pull/2119))
 - spl: Add `MetadataAccount` account deserialization. ([#2014](https://github.com/coral-xyz/anchor/pull/2014)).

cli/src/config.rs

@@ -341,8 +341,12 @@ pub struct BuildConfig {
 }
 
 impl Config {
-    pub fn add_test_config(&mut self, root: impl AsRef<Path>) -> Result<()> {
+    pub fn add_test_config(
-        self.test_config = TestConfig::discover(root)?;
+        &mut self,
+        root: impl AsRef<Path>,
+        test_paths: Vec<PathBuf>,
+    ) -> Result<()> {
+        self.test_config = TestConfig::discover(root, test_paths)?;
         Ok(())
     }
 
@@ -662,14 +666,17 @@ impl Deref for TestConfig {
 }
 
 impl TestConfig {
-    pub fn discover(root: impl AsRef<Path>) -> Result<Option<Self>> {
+    pub fn discover(root: impl AsRef<Path>, test_paths: Vec<PathBuf>) -> Result<Option<Self>> {
         let walker = WalkDir::new(root).into_iter();
         let mut test_suite_configs = HashMap::new();
         for entry in walker.filter_entry(|e| !is_hidden(e)) {
             let entry = entry?;
             if entry.file_name() == "Test.toml" {
-                let test_toml = TestToml::from_path(entry.path())?;
+                let entry_path = entry.path();
-                test_suite_configs.insert(entry.path().into(), test_toml);
+                let test_toml = TestToml::from_path(entry_path)?;
+                if test_paths.is_empty() || test_paths.iter().any(|p| entry_path.starts_with(p)) {
+                    test_suite_configs.insert(entry.path().into(), test_toml);
+                }
             }
         }
 

cli/src/lib.rs

@@ -167,6 +167,9 @@ pub enum Command {
         /// to be able to check the transactions.
         #[clap(long)]
         detach: bool,
+        /// Run the test suites under the specified path
+        #[clap(long)]
+        run: Vec<String>,
         args: Vec<String>,
         /// Arguments to pass to the underlying `cargo build-bpf` command.
         #[clap(required = false, last = true)]
@@ -426,6 +429,7 @@ pub fn entry(opts: Opts) -> Result<()> {
             skip_local_validator,
             skip_build,
             detach,
+            run,
             args,
             cargo_args,
             skip_lint,
@@ -436,6 +440,7 @@ pub fn entry(opts: Opts) -> Result<()> {
             skip_build,
             skip_lint,
             detach,
+            run,
             args,
             cargo_args,
         ),
@@ -1863,9 +1868,19 @@ fn test(
     skip_build: bool,
     skip_lint: bool,
     detach: bool,
+    tests_to_run: Vec<String>,
     extra_args: Vec<String>,
     cargo_args: Vec<String>,
 ) -> Result<()> {
+    let test_paths = tests_to_run
+        .iter()
+        .map(|path| {
+            PathBuf::from(path)
+                .canonicalize()
+                .map_err(|_| anyhow!("Wrong path {}", path))
+        })
+        .collect::<Result<Vec<_>, _>>()?;
+
     with_workspace(cfg_override, |cfg| {
         // Build if needed.
         if !skip_build {
@@ -1887,7 +1902,7 @@ fn test(
         }
 
         let root = cfg.path().parent().unwrap().to_owned();
-        cfg.add_test_config(root)?;
+        cfg.add_test_config(root, test_paths)?;
 
         // Run the deploy against the cluster in two cases:
         //

tests/multiple-suites-run-single/Anchor.toml

@@ -0,0 +1,14 @@
+[features]
+seeds = false
+[programs.localnet]
+multiple_suites_run_single = "Fg6PaFpoGXkYsidMpWTK6W2BeZ7FEfcYkg476zPFsLnS"
+
+[registry]
+url = "https://anchor.projectserum.com"
+
+[provider]
+cluster = "localnet"
+wallet = "~/.config/solana/id.json"
+
+[test]
+startup_wait = 20000

tests/multiple-suites-run-single/Cargo.toml

@@ -0,0 +1,4 @@
+[workspace]
+members = [
+    "programs/*"
+]

tests/multiple-suites-run-single/package.json

@@ -0,0 +1,19 @@
+{
+    "name": "multiple-suites-run-single",
+    "version": "0.24.2",
+    "license": "(MIT OR Apache-2.0)",
+    "homepage": "https://github.com/project-serum/anchor#readme",
+    "bugs": {
+      "url": "https://github.com/project-serum/anchor/issues"
+    },
+    "repository": {
+      "type": "git",
+      "url": "https://github.com/project-serum/anchor.git"
+    },
+    "engines": {
+      "node": ">=11"
+    },
+    "scripts": {
+      "test": "anchor test"
+    }
+  }

tests/multiple-suites-run-single/programs/multiple-suites-run-single/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "multiple-suites-run-single"
+version = "0.1.0"
+description = "Created with Anchor"
+edition = "2018"
+
+[lib]
+crate-type = ["cdylib", "lib"]
+name = "multiple_suites_run_single"
+
+[features]
+no-entrypoint = []
+no-idl = []
+no-log-ix-name = []
+cpi = ["no-entrypoint"]
+default = []
+
+[dependencies]
+anchor-lang = { path = "../../../../lang" }

tests/multiple-suites-run-single/programs/multiple-suites-run-single/Xargo.toml

@@ -0,0 +1,2 @@
+[target.bpfel-unknown-unknown.dependencies.std]
+features = []

tests/multiple-suites-run-single/programs/multiple-suites-run-single/src/lib.rs

@@ -0,0 +1,15 @@
+use anchor_lang::prelude::*;
+
+declare_id!("Fg6PaFpoGXkYsidMpWTK6W2BeZ7FEfcYkg476zPFsLnS");
+
+#[program]
+pub mod multiple_suites_run_single {
+    use super::*;
+
+    pub fn initialize(_ctx: Context<Initialize>) -> Result<()> {
+        Ok(())
+    }
+}
+
+#[derive(Accounts)]
+pub struct Initialize {}

tests/multiple-suites-run-single/tests/should-not-run/Test.toml

@@ -0,0 +1,4 @@
+extends = ["../../Anchor.toml"]
+
+[scripts]
+test = "yarn run ts-mocha -p ./tsconfig.json -t 1000000 tests/should-not-run/**/*.ts"

tests/multiple-suites-run-single/tests/should-not-run/shouldNotRun.ts

@@ -0,0 +1,5 @@
+describe("multiple-suites-run-single", () => {
+  it("Should not be executed", async () => {
+    throw new Error("This test has to be skipped");
+  });
+});

tests/multiple-suites-run-single/tests/should-run/Test.toml

@@ -0,0 +1,4 @@
+extends = ["../../Anchor.toml"]
+
+[scripts]
+test = "yarn run ts-mocha -p ./tsconfig.json -t 1000000 tests/should-run/**/*.ts"

tests/multiple-suites-run-single/tests/should-run/shouldRun.ts

@@ -0,0 +1,17 @@
+import * as anchor from "@project-serum/anchor";
+import { Program } from "@project-serum/anchor";
+import { assert } from "chai";
+import { MultipleSuitesRunSingle } from "../../target/types/multiple_suites_run_single";
+
+describe("multiple-suites-run-single", () => {
+  // Configure the client to use the local cluster.
+  anchor.setProvider(anchor.AnchorProvider.env());
+
+  const program = anchor.workspace
+    .MultipleSuitesRunSingle as Program<MultipleSuitesRunSingle>;
+
+  it("Is initialized!", async () => {
+    const tx = await program.methods.initialize().rpc();
+    console.log("Your transaction signature", tx);
+  });
+});

tests/multiple-suites-run-single/tsconfig.json

@@ -0,0 +1,11 @@
+{
+  "compilerOptions": {
+    "types": ["mocha", "chai"],
+    "typeRoots": ["./node_modules/@types"],
+    "lib": ["es2015"],
+    "module": "commonjs",
+    "target": "es6",
+    "esModuleInterop": true,
+    "skipLibCheck": true
+  }
+}

tests/package.json

@@ -37,6 +37,7 @@
     "declare-id",
     "cpi-returns",
     "multiple-suites",
+    "multiple-suites-run-single",
     "bpf-upgradeable-state"
   ],
   "dependencies": {