From 7e4feb7d8bd56025d3077ebd9983c33d94d723c2 Mon Sep 17 00:00:00 2001 From: Luni-4 Date: Mon, 14 Aug 2023 23:43:21 +0200 Subject: [PATCH] Convert `run-checks` script into a Rust binary (#628) --- .github/pull_request_template.md | 2 +- .gitignore | 6 +- README.md | 34 +++- run-checks.ps1 | 20 +++ run-checks.sh | 160 +++--------------- scripts/run-checks.rs | 274 +++++++++++++++++++++++++++++++ 6 files changed, 352 insertions(+), 144 deletions(-) create mode 100644 run-checks.ps1 create mode 100644 scripts/run-checks.rs diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index fe09f1173..25042bc38 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -2,7 +2,7 @@ ### Checklist -- [ ] Confirm that `run-checks.sh` has been executed. +- [ ] Confirm that `run-checks` script has been executed. ### Related Issues/PRs diff --git a/.gitignore b/.gitignore index c97e4a452..680247423 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,8 @@ Cargo.lock .cargo/config.toml .idea -.vscode \ No newline at end of file +.vscode + +# Ignore binaries contained in the `scripts` directory +scripts/publish +scripts/run-checks diff --git a/README.md b/README.md index 5074f5749..990e538a8 100644 --- a/README.md +++ b/README.md @@ -285,7 +285,30 @@ recommended to read our [architecture document](https://github.com/burn-rs/burn/tree/main/ARCHITECTURE.md), which explains our architectural decisions. Please see more details in our [contributing guide](/CONTRIBUTING.md). -## CI +## Continuous Integration + +### Run checks + +On Unix systems, run `run-checks.sh` using this command + +``` +run-checks.sh environment +``` + +On Windows systems, run `run-checks.ps1` using this command: + +``` +run-checks.ps1 environment +``` + +The `environment` argument can assume **ONLY** the following values: + +- `std` to perform checks using `libstd` +- `no_std` to perform checks on an embedded environment using `libcore` + +If no `environment` value has been passed, run both `std` and `no_std` checks. + +## Continuous Deployment ### Publish crates @@ -295,6 +318,15 @@ Compile `scripts/publish.rs` using this command: rustc scripts/publish.rs --crate-type bin --out-dir scripts ``` +Run `scripts/publish` using this command + +``` +./scripts/publish crate_name +``` + +where `crate_name` is the name of the crate to publish + + ## Disclaimer Burn is currently in active development, and there will be breaking changes. While any resulting diff --git a/run-checks.ps1 b/run-checks.ps1 new file mode 100644 index 000000000..985ef8cf9 --- /dev/null +++ b/run-checks.ps1 @@ -0,0 +1,20 @@ +# This script runs all `burn` checks locally +# +# Run `run-checks` using this command: +# +# ./scripts/run-checks environment +# +# where `environment` can assume **ONLY** the following values: +# +# - `std` to perform checks using `libstd` +# - `no_std` to perform checks on an embedded environment using `libcore` +# +# If no `environment` value has been passed, run both `std` and `no_std` checks. + +# Compile run-checks binary +rustc scripts/run-checks.rs --crate-type bin --out-dir scripts + +# Run binary passing the first input parameter, who is mandatory. +# If the input parameter is missing or wrong, it will be the `run-checks` +# binary which will be responsible of arising an error. +./scripts/run-checks $args[0] diff --git a/run-checks.sh b/run-checks.sh index d9269f05b..e3e9c7d4c 100755 --- a/run-checks.sh +++ b/run-checks.sh @@ -1,144 +1,22 @@ #!/bin/bash +# +# This script runs all `burn` checks locally +# +# Run `run-checks` using this command: +# +# ./scripts/run-checks environment +# +# where `environment` can assume **ONLY** the following values: +# +# - `std` to perform checks using `libstd` +# - `no_std` to perform checks on an embedded environment using `libcore` +# +# If no `environment` value has been passed, run both `std` and `no_std` checks. -# This script is run before a PR is created. -# It is used to check that the code compiles and passes all tests. -# It is also used to check that the code is formatted correctly and passes clippy. +# Compile run-checks binary +rustc scripts/run-checks.rs --crate-type bin --out-dir scripts -# Usage: ./run-checks.sh {all|no_std|std} (default: all) - -# Exit immediately if a command exits with a non-zero status. -set -euo pipefail - -# Function to handle errors -error_handler() { - local exit_status=$? - local line_number=$1 - local command=$2 - - echo "Error on line $line_number" - echo "Command '$command' exited with status $exit_status" -} - -# Signal trap to call error_handler when a command fails -trap 'error_handler $LINENO $BASH_COMMAND' ERR - -# Function to build and test no_std -build_and_test_no_std() { - local dir=$1 - - echo "$dir" - cd $dir || exit - - echo "Build without defaults" - cargo build --no-default-features - - echo "Test without defaults" - cargo test --no-default-features - - echo "Build for WebAssembly" - cargo build --no-default-features --target wasm32-unknown-unknown - - echo "Build for ARM" - cargo build --no-default-features --target thumbv7m-none-eabi - - cd .. || exit -} - -# Function to build and test all features -build_and_test_all_features() { - local dir=$1 - - echo "$dir" - cd $dir || exit - - echo "Build with all defaults" - cargo build --all-features - - echo "Test with all features" - cargo test --all-features - - echo "Check documentation with all features" - cargo doc --all-features - - cd .. || exit -} - -# Set RUSTDOCFLAGS to treat warnings as errors for the documentation build -export RUSTDOCFLAGS="-D warnings" - -# Run the checks for std and all features with std -std_func() { - echo "Running std checks" - - cargo build --workspace - cargo test --workspace - cargo fmt --check --all - cargo clippy -- -D warnings - cargo doc --workspace - - # all features - echo "Running all-features checks" - build_and_test_all_features "burn-dataset" - - cd burn-core || exit - - echo "Test burn-core with tch backend" - cargo test --features test-tch - - echo "Test burn-core with wgpu backend" - cargo test --features test-wgpu - - cd .. || exit -} - -# Run the checks for no_std -no_std_func() { - echo "Running no_std checks" - - # Add wasm32 target for compiler. - rustup target add wasm32-unknown-unknown - rustup target add thumbv7m-none-eabi - - build_and_test_no_std "burn" - build_and_test_no_std "burn-core" - build_and_test_no_std "burn-common" - build_and_test_no_std "burn-tensor" - build_and_test_no_std "burn-ndarray" - build_and_test_no_std "burn-no-std-tests" -} - -# Save the script start time -start_time=$(date +%s) - -# If no arguments were supplied or if it's empty, set the default as 'all' -if [ -z "${1-}" ]; then - arg="all" -else - arg=$1 -fi - -# Check the argument and call the appropriate functions -case $arg in -all) - no_std_func - std_func - ;; -no_std) - no_std_func - ;; -std) - std_func - ;; -*) - echo "Error: Invalid argument" - echo "Usage: $0 {all|no_std|std}" - exit 1 - ;; -esac - -# Calculate and print the script execution time -end_time=$(date +%s) -execution_time=$((end_time - start_time)) -echo "Script executed in $execution_time seconds." - -exit 0 +# Run binary passing the first input parameter, who is mandatory. +# If the input parameter is missing or wrong, it will be the `run-checks` +# binary which will be responsible of arising an error. +./scripts/run-checks $1 diff --git a/scripts/run-checks.rs b/scripts/run-checks.rs new file mode 100644 index 000000000..e415b640e --- /dev/null +++ b/scripts/run-checks.rs @@ -0,0 +1,274 @@ +//! This script is run before a PR is created. +//! +//! It is used to check that the code compiles and passes all tests. +//! +//! It is also used to check that the code is formatted correctly and passes clippy. +//! +//! To build this script, run the following command: +//! +//! rustc scripts/run-checks.rs --crate-type bin --out-dir scripts +//! +//! To run the script: +//! +//! ./scripts/run-checks environment +//! +//! where `environment` can assume **ONLY** the following values: +//! - `std` to perform checks using `libstd` +//! - `no_std` to perform checks on an embedded environment using `libcore` + +use std::env; +use std::process::{Child, Command, Stdio}; +use std::str; +use std::time::Instant; + +// Targets constants +const WASM32_TARGET: &str = "wasm32-unknown-unknown"; +const ARM_TARGET: &str = "thumbv7m-none-eabi"; + +// Handle child process +fn handle_child_process(mut child: Child, error: &str) { + // Wait for the child process to finish + let status = child.wait().expect(error); + + // If exit status is not a success, terminate the process with an error + if !status.success() { + // Use the exit code associated to a command to terminate the process, + // if any exit code had been found, use the default value 1 + std::process::exit(status.code().unwrap_or(1)); + } +} + +// Define and run rustup command +fn rustup(target: &str) { + // Rustup arguments + let args = ["target", "add", target]; + + // Print rustup command + println!("rustup {}\n\n", args.join(" ")); + + // Run rustup command as child process + let rustup = Command::new("rustup") + .args(args) + .stdout(Stdio::inherit()) // Send stdout directly to terminal + .stderr(Stdio::inherit()) // Send stderr directly to terminal + .spawn() + .expect("Failed to run rustup"); + + // Handle rustup child process + handle_child_process(rustup, "Failed to wait for rustup child process"); +} + +// Define and run a cargo command +fn run_cargo(command: &str, first_params: &[&str], second_params: &[&str], error: &str) { + // Print cargo command + println!( + "\ncargo {} {} {}\n", + command, + first_params.join(" "), + second_params.join(" ") + ); + + // Run cargo + let cargo = Command::new("cargo") + .arg(command) + .args(first_params) + .args(second_params) + .stdout(Stdio::inherit()) // Send stdout directly to terminal + .stderr(Stdio::inherit()) // Send stderr directly to terminal + .spawn() + .expect(error); + + // Handle cargo child process + handle_child_process(cargo, "Failed to wait for cargo child process"); +} + +// Run cargo build command +fn cargo_build(params: &[&str]) { + // Run cargo build + run_cargo( + "build", + params, + &["--color=always"], + "Failed to run cargo build", + ); +} + +// Run cargo test command +fn cargo_test(params: &[&str]) { + // Run cargo test + run_cargo( + "test", + params, + &["--color=always", "--", "--color=always"], + "Failed to run cargo test", + ); +} + +// Run cargo fmt command +fn cargo_fmt() { + // Run cargo fmt + run_cargo( + "fmt", + &["--check", "--all"], + &["--", "--color=always"], + "Failed to run cargo fmt", + ); +} + +// Run cargo clippy command +fn cargo_clippy() { + // Run cargo clippy + run_cargo( + "clippy", + &["--color=always"], + &["--", "-D", "warnings"], + "Failed to run cargo clippy", + ); +} + +// Run cargo doc command +fn cargo_doc(params: &[&str]) { + // Run cargo doc + run_cargo( + "doc", + params, + &["--color=always"], + "Failed to run cargo doc", + ); +} + +// Build and test a crate in a no_std environment +fn build_and_test_no_std(crate_name: &str) { + println!("\nRun checks for `{}` crate", crate_name); + + // Run cargo build --no-default-features + cargo_build(&["-p", crate_name, "--no-default-features"]); + + // Run cargo test --no-default-features + cargo_test(&["-p", crate_name, "--no-default-features"]); + + // Run cargo build --no-default-features --target wasm32-unknown-unknowns + cargo_build(&[ + "-p", + crate_name, + "--no-default-features", + "--target", + WASM32_TARGET, + ]); + + // Run cargo build --no-default-features --target thumbv7m-none-eabi + cargo_build(&[ + "-p", + crate_name, + "--no-default-features", + "--target", + ARM_TARGET, + ]); +} + +// Run no_std checks +fn no_std_checks() { + println!("Checks for no_std environment...\n\n"); + + // Install wasm32 target + rustup(WASM32_TARGET); + + // Install ARM target + rustup(ARM_TARGET); + + // Run checks for the following crates + build_and_test_no_std("burn"); + build_and_test_no_std("burn-core"); + build_and_test_no_std("burn-common"); + build_and_test_no_std("burn-tensor"); + build_and_test_no_std("burn-ndarray"); + build_and_test_no_std("burn-no-std-tests"); +} + +// Test burn-core with tch and wgpu backend +fn burn_core_std() { + println!("\n\nRun checks for burn-core crate with tch and wgpu backend"); + + // Run cargo test --features test-tch + cargo_test(&["-p", "burn-core", "--features", "test-tch"]); + + // Run cargo test --features test-wgpu + cargo_test(&["-p", "burn-core", "--features", "test-wgpu"]); +} + +// Test burn-dataset features +fn burn_dataset_features_std() { + println!("\n\nRun checks for burn-dataset features"); + + // Run cargo build --all-features + cargo_build(&["-p", "burn-dataset", "--all-features"]); + + // Run cargo test --all-features + cargo_test(&["-p", "burn-dataset", "--all-features"]); + + // Run cargo doc --all-features + cargo_doc(&["-p", "burn-dataset", "--all-features"]); +} + +fn std_checks() { + // Set RUSTDOCFLAGS environment variable to treat warnings as errors + // for the documentation build + env::set_var("RUSTDOCFLAGS", "-D warnings"); + + println!("Running std checks"); + + // Build each workspace + cargo_build(&["--workspace"]); + + // Test each workspace + cargo_test(&["--workspace"]); + + // Check format + cargo_fmt(); + + // Check clippy lints + cargo_clippy(); + + // Produce documentation for each workspace + cargo_doc(&["--workspace"]); + + // Test burn-dataset features + burn_dataset_features_std(); + + // Test burn-core with tch and wgpu backend + burn_core_std(); +} + +fn main() { + // Start time measurement + let start = Instant::now(); + + // The environment can assume ONLY "std" and "no_std" as values. + // + // Depending on the input argument, the respective environment checks + // are run. + // + // If no environment has been passed, run both "std" and "no_std" checks. + match env::args() + .nth( + 1, /* Index of the first argument, because 0 is the binary name */ + ) + .as_deref() + { + Some("std") => std_checks(), + Some("no_std") => no_std_checks(), + Some(_) | None => { + /* Run both "std" and "no_std" checks" */ + std_checks(); + no_std_checks(); + } + } + + // Stop time measurement + // + // Compute runtime duration + let duration = start.elapsed(); + + // Print duration + println!("Time elapsed for the current execution: {:?}", duration); +}