mirror of https://github.com/tracel-ai/burn.git
chore(ci): CI grouping and refactoring (#1024)
This commit is contained in:
David Chavez
GitHub
parent
8fc52113bc
commit
82f5722ca1
|
|
@ -116,7 +116,7 @@ jobs:
|
|||
tar xj -C $HOME/.cargo/bin
|
||||
|
||||
- name: run checks & tests
|
||||
run: ${{ matrix.coverage-flags }} ${{ matrix.wgpu-flags }} CI_RUN=1 cargo xtask run-checks ${{ matrix.test }}
|
||||
run: ${{ matrix.coverage-flags }} ${{ matrix.wgpu-flags }} cargo xtask run-checks ${{ matrix.test }}
|
||||
|
||||
- name: Codecov upload
|
||||
if: matrix.rust == 'stable' && matrix.test == 'std' && runner.os == 'Linux'
|
||||
|
|
@ -145,4 +145,4 @@ jobs:
|
|||
tar xz -C $HOME/.cargo/bin
|
||||
|
||||
- name: run spelling checks using typos
|
||||
run: CI_RUN=1 cargo xtask run-checks typos
|
||||
run: cargo xtask run-checks typos
|
||||
|
|
|
|||
|
|
@ -9,3 +9,6 @@ license = "MIT OR Apache-2.0"
|
|||
[dependencies]
|
||||
anyhow = "1.0.75"
|
||||
clap = { version = "4.4.8", features = ["derive"] }
|
||||
env_logger = "0.10.0"
|
||||
log = "0.4.17"
|
||||
serde_json = { version = "1" }
|
||||
|
|
|
|||
|
|
@ -0,0 +1,67 @@
|
|||
use std::io::Write;
|
||||
|
||||
/// Initialise and create a `env_logger::Builder` which follows the
|
||||
/// GitHub Actions logging syntax when running on CI.
|
||||
pub fn init_logger() -> env_logger::Builder {
|
||||
let mut builder = env_logger::Builder::from_default_env();
|
||||
builder.target(env_logger::Target::Stdout);
|
||||
|
||||
// Find and setup the correct log level
|
||||
builder.filter(None, get_log_level());
|
||||
builder.write_style(env_logger::WriteStyle::Always);
|
||||
|
||||
// Custom Formatter for Github Actions
|
||||
if std::env::var("CI").is_ok() {
|
||||
builder.format(|buf, record| match record.level().as_str() {
|
||||
"DEBUG" => writeln!(buf, "::debug:: {}", record.args()),
|
||||
"WARN" => writeln!(buf, "::warning:: {}", record.args()),
|
||||
"ERROR" => {
|
||||
writeln!(buf, "::error:: {}", record.args())
|
||||
}
|
||||
_ => writeln!(buf, "{}", record.args()),
|
||||
});
|
||||
}
|
||||
|
||||
builder
|
||||
}
|
||||
|
||||
/// Determine the LogLevel for the logger
|
||||
fn get_log_level() -> log::LevelFilter {
|
||||
// DEBUG
|
||||
match std::env::var("DEBUG") {
|
||||
Ok(_value) => return log::LevelFilter::Debug,
|
||||
Err(_err) => (),
|
||||
}
|
||||
// ACTIONS_RUNNER_DEBUG
|
||||
match std::env::var("ACTIONS_RUNNER_DEBUG") {
|
||||
Ok(_value) => return log::LevelFilter::Debug,
|
||||
Err(_err) => (),
|
||||
};
|
||||
|
||||
log::LevelFilter::Info
|
||||
}
|
||||
|
||||
/// Group Macro
|
||||
#[macro_export]
|
||||
macro_rules! group {
|
||||
// group!()
|
||||
($($arg:tt)*) => {
|
||||
let title = format!($($arg)*);
|
||||
if std::env::var("CI").is_ok() {
|
||||
log!(log::Level::Info, "::group::{}", title)
|
||||
} else {
|
||||
log!(log::Level::Info, "{}", title)
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// End Group Macro
|
||||
#[macro_export]
|
||||
macro_rules! endgroup {
|
||||
// endgroup!()
|
||||
() => {
|
||||
if std::env::var("CI").is_ok() {
|
||||
log!(log::Level::Info, "::endgroup::")
|
||||
}
|
||||
};
|
||||
}
|
||||
|
|
@ -1,7 +1,12 @@
|
|||
use clap::{Parser, Subcommand};
|
||||
|
||||
mod logging;
|
||||
mod publish;
|
||||
mod runchecks;
|
||||
mod utils;
|
||||
|
||||
#[macro_use]
|
||||
extern crate log;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(author, version, about, long_about = None)]
|
||||
|
|
|
|||
|
|
@ -4,7 +4,11 @@
|
|||
//!
|
||||
//! It is also used to check that the code is formatted correctly and passes clippy.
|
||||
|
||||
use crate::logging::init_logger;
|
||||
use crate::utils::{format_duration, get_workspaces, WorkspaceMemberType};
|
||||
use crate::{endgroup, group};
|
||||
use std::env;
|
||||
use std::path::Path;
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::str;
|
||||
use std::time::Instant;
|
||||
|
|
@ -29,7 +33,7 @@ fn handle_child_process(mut child: Child, error: &str) {
|
|||
// Run a command
|
||||
fn run_command(command: &str, args: &[&str], command_error: &str, child_error: &str) {
|
||||
// Format command
|
||||
println!("{command} {}\n\n", args.join(" "));
|
||||
info!("{command} {}\n\n", args.join(" "));
|
||||
|
||||
// Run command as child process
|
||||
let command = Command::new(command)
|
||||
|
|
@ -45,31 +49,48 @@ fn run_command(command: &str, args: &[&str], command_error: &str, child_error: &
|
|||
|
||||
// Define and run rustup command
|
||||
fn rustup(command: &str, target: &str) {
|
||||
group!("Rustup: {} add {}", command, target);
|
||||
run_command(
|
||||
"rustup",
|
||||
&[command, "add", target],
|
||||
"Failed to run rustup",
|
||||
"Failed to wait for rustup child process",
|
||||
)
|
||||
);
|
||||
endgroup!();
|
||||
}
|
||||
|
||||
// Define and run a cargo command
|
||||
fn run_cargo(command: &str, params: Params, error: &str) {
|
||||
run_cargo_with_path::<String>(command, params, None, error)
|
||||
}
|
||||
|
||||
// Define and run a cargo command with curr dir
|
||||
fn run_cargo_with_path<P: AsRef<Path>>(
|
||||
command: &str,
|
||||
params: Params,
|
||||
path: Option<P>,
|
||||
error: &str,
|
||||
) {
|
||||
// Print cargo command
|
||||
println!("\ncargo {} {}\n", command, params);
|
||||
info!("cargo {} {}\n", command, params);
|
||||
|
||||
// Run cargo
|
||||
let cargo = Command::new("cargo")
|
||||
let mut cargo = Command::new("cargo");
|
||||
cargo
|
||||
.env("CARGO_INCREMENTAL", "0")
|
||||
.arg(command)
|
||||
.args(params.params)
|
||||
.stdout(Stdio::inherit()) // Send stdout directly to terminal
|
||||
.stderr(Stdio::inherit()) // Send stderr directly to terminal
|
||||
.spawn()
|
||||
.expect(error);
|
||||
.stderr(Stdio::inherit()); // Send stderr directly to terminal
|
||||
|
||||
if let Some(path) = path {
|
||||
cargo.current_dir(path);
|
||||
}
|
||||
|
||||
let cargo_process = cargo.spawn().expect(error);
|
||||
|
||||
// Handle cargo child process
|
||||
handle_child_process(cargo, "Failed to wait for cargo child process");
|
||||
handle_child_process(cargo_process, "Failed to wait for cargo child process");
|
||||
}
|
||||
|
||||
// Run cargo build command
|
||||
|
|
@ -104,17 +125,18 @@ fn cargo_test(params: Params) {
|
|||
|
||||
// Run cargo fmt command
|
||||
fn cargo_fmt() {
|
||||
// Run cargo fmt
|
||||
group!("Cargo: fmt");
|
||||
run_cargo(
|
||||
"fmt",
|
||||
["--check", "--all", "--", "--color=always"].into(),
|
||||
"Failed to run cargo fmt",
|
||||
);
|
||||
endgroup!();
|
||||
}
|
||||
|
||||
// Run cargo clippy command
|
||||
fn cargo_clippy() {
|
||||
if std::env::var("CI_RUN").is_ok() {
|
||||
if std::env::var("CI").is_ok() {
|
||||
return;
|
||||
}
|
||||
// Run cargo clippy
|
||||
|
|
@ -133,7 +155,7 @@ fn cargo_doc(params: Params) {
|
|||
|
||||
// Build and test a crate in a no_std environment
|
||||
fn build_and_test_no_std<const N: usize>(crate_name: &str, extra_args: [&str; N]) {
|
||||
println!("\nRun checks for `{}` crate", crate_name);
|
||||
group!("Checks: {} (no-std)", crate_name);
|
||||
|
||||
// Run cargo build --no-default-features
|
||||
cargo_build(Params::from(["-p", crate_name, "--no-default-features"]) + extra_args);
|
||||
|
|
@ -162,6 +184,8 @@ fn build_and_test_no_std<const N: usize>(crate_name: &str, extra_args: [&str; N]
|
|||
ARM_TARGET,
|
||||
]) + extra_args,
|
||||
);
|
||||
|
||||
endgroup!();
|
||||
}
|
||||
|
||||
// Setup code coverage
|
||||
|
|
@ -201,8 +225,6 @@ fn run_grcov() {
|
|||
|
||||
// Run no_std checks
|
||||
fn no_std_checks() {
|
||||
println!("Checks for no_std environment...\n\n");
|
||||
|
||||
// Install wasm32 target
|
||||
rustup("target", WASM32_TARGET);
|
||||
|
||||
|
|
@ -224,20 +246,22 @@ fn no_std_checks() {
|
|||
|
||||
// Test burn-core with tch and wgpu backend
|
||||
fn burn_core_std() {
|
||||
println!("\n\nRun checks for burn-core crate with tch and wgpu backend");
|
||||
|
||||
// Run cargo test --features test-tch
|
||||
group!("Test: burn-core (tch)");
|
||||
cargo_test(["-p", "burn-core", "--features", "test-tch"].into());
|
||||
endgroup!();
|
||||
|
||||
// Run cargo test --features test-wgpu
|
||||
if std::env::var("DISABLE_WGPU").is_err() {
|
||||
group!("Test: burn-core (wgpu)");
|
||||
cargo_test(["-p", "burn-core", "--features", "test-wgpu"].into());
|
||||
endgroup!();
|
||||
}
|
||||
}
|
||||
|
||||
// Test burn-dataset features
|
||||
fn burn_dataset_features_std() {
|
||||
println!("\n\nRun checks for burn-dataset features");
|
||||
group!("Checks: burn-dataset (all-features)");
|
||||
|
||||
// Run cargo build --all-features
|
||||
cargo_build(["-p", "burn-dataset", "--all-features"].into());
|
||||
|
|
@ -247,13 +271,17 @@ fn burn_dataset_features_std() {
|
|||
|
||||
// Run cargo doc --all-features
|
||||
cargo_doc(["-p", "burn-dataset", "--all-features"].into());
|
||||
|
||||
endgroup!();
|
||||
}
|
||||
|
||||
// Test burn-candle with accelerate (macOS only)
|
||||
// Leverages the macOS Accelerate framework: https://developer.apple.com/documentation/accelerate
|
||||
#[cfg(target_os = "macos")]
|
||||
fn burn_candle_accelerate() {
|
||||
group!("Checks: burn-candle (accelerate)");
|
||||
cargo_test(["-p", "burn-candle", "--features", "accelerate"].into());
|
||||
endgroup!();
|
||||
}
|
||||
|
||||
fn std_checks() {
|
||||
|
|
@ -265,31 +293,38 @@ fn std_checks() {
|
|||
let is_coverage = std::env::var("COVERAGE").is_ok();
|
||||
let disable_wgpu = std::env::var("DISABLE_WGPU").is_ok();
|
||||
|
||||
println!("Running std checks");
|
||||
|
||||
// Check format
|
||||
cargo_fmt();
|
||||
|
||||
// Check clippy lints
|
||||
cargo_clippy();
|
||||
|
||||
// Build each workspace
|
||||
if disable_wgpu {
|
||||
cargo_build(["--workspace", "--exclude=xtask", "--exclude=burn-wgpu"].into());
|
||||
} else {
|
||||
cargo_build(["--workspace", "--exclude=xtask"].into());
|
||||
}
|
||||
|
||||
// Produce documentation for each workspace
|
||||
group!("Docs: workspaces");
|
||||
cargo_doc(["--workspace"].into());
|
||||
endgroup!();
|
||||
|
||||
// Setup code coverage
|
||||
if is_coverage {
|
||||
setup_coverage();
|
||||
}
|
||||
|
||||
// Test each workspace
|
||||
cargo_test(["--workspace"].into());
|
||||
// Build & test each workspace
|
||||
let workspaces = get_workspaces(WorkspaceMemberType::Crate);
|
||||
for workspace in workspaces {
|
||||
if disable_wgpu && workspace.name == "burn-wgpu" {
|
||||
continue;
|
||||
}
|
||||
|
||||
if workspace.name == "burn-tch" {
|
||||
continue;
|
||||
}
|
||||
|
||||
group!("Checks: {}", workspace.name);
|
||||
cargo_build(Params::from(["-p", &workspace.name]));
|
||||
cargo_test(Params::from(["-p", &workspace.name]));
|
||||
endgroup!();
|
||||
}
|
||||
|
||||
// Test burn-candle with accelerate (macOS only)
|
||||
#[cfg(target_os = "macos")]
|
||||
|
|
@ -316,12 +351,12 @@ fn check_typos() {
|
|||
|
||||
// Do not run cargo install on CI to speed up the computation.
|
||||
// Check whether the file has been installed on
|
||||
if std::env::var("CI_RUN").is_err() && !typos_cli_path.exists() {
|
||||
if std::env::var("CI").is_err() && !typos_cli_path.exists() {
|
||||
// Install typos-cli
|
||||
cargo_install(["typos-cli", "--version", "1.16.5"].into());
|
||||
}
|
||||
|
||||
println!("Running typos check \n\n");
|
||||
info!("Running typos check \n\n");
|
||||
|
||||
// Run typos command as child process
|
||||
let typos = Command::new("typos")
|
||||
|
|
@ -335,34 +370,21 @@ fn check_typos() {
|
|||
}
|
||||
|
||||
fn check_examples() {
|
||||
println!("Checking examples compile \n\n");
|
||||
|
||||
std::fs::read_dir("examples").unwrap().for_each(|dir| {
|
||||
let dir = dir.unwrap();
|
||||
let path = dir.path();
|
||||
// Skip if not a directory
|
||||
if !path.is_dir() {
|
||||
return;
|
||||
let workspaces = get_workspaces(WorkspaceMemberType::Example);
|
||||
for workspace in workspaces {
|
||||
if workspace.name == "notebook" {
|
||||
continue;
|
||||
}
|
||||
if path.file_name().unwrap().to_str().unwrap() == "notebook" {
|
||||
// not a crate
|
||||
return;
|
||||
}
|
||||
let path = path.to_str().unwrap();
|
||||
println!("Checking {path} \n\n");
|
||||
|
||||
let child = Command::new("cargo")
|
||||
.arg("check")
|
||||
.arg("--examples")
|
||||
.current_dir(dir.path())
|
||||
.stdout(Stdio::inherit()) // Send stdout directly to terminal
|
||||
.stderr(Stdio::inherit()) // Send stderr directly to terminal
|
||||
.spawn()
|
||||
.expect("Failed to check examples");
|
||||
|
||||
// Handle typos child process
|
||||
handle_child_process(child, "Failed to wait for examples child process");
|
||||
});
|
||||
group!("Checks: Example - {}", workspace.name);
|
||||
run_cargo_with_path(
|
||||
"check",
|
||||
["--examples"].into(),
|
||||
Some(workspace.path),
|
||||
"Failed to check example",
|
||||
);
|
||||
endgroup!();
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(clap::ValueEnum, Default, Copy, Clone, PartialEq, Eq)]
|
||||
|
|
@ -381,6 +403,9 @@ pub enum CheckType {
|
|||
}
|
||||
|
||||
pub fn run(env: CheckType) -> anyhow::Result<()> {
|
||||
// Setup logger
|
||||
init_logger().init();
|
||||
|
||||
// Start time measurement
|
||||
let start = Instant::now();
|
||||
|
||||
|
|
@ -411,7 +436,10 @@ pub fn run(env: CheckType) -> anyhow::Result<()> {
|
|||
let duration = start.elapsed();
|
||||
|
||||
// Print duration
|
||||
println!("Time elapsed for the current execution: {:?}", duration);
|
||||
info!(
|
||||
"\x1B[32;1mTime elapsed for the current execution: {}\x1B[0m",
|
||||
format_duration(&duration)
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,81 @@
|
|||
use serde_json::Value;
|
||||
use std::{process::Command, time::Duration};
|
||||
|
||||
pub(crate) enum WorkspaceMemberType {
|
||||
Crate,
|
||||
Example,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct WorkspaceMember {
|
||||
pub(crate) name: String,
|
||||
pub(crate) path: String,
|
||||
}
|
||||
|
||||
impl WorkspaceMember {
|
||||
fn new(name: String, path: String) -> Self {
|
||||
Self { name, path }
|
||||
}
|
||||
}
|
||||
|
||||
/// Get project workspaces
|
||||
pub(crate) fn get_workspaces(w_type: WorkspaceMemberType) -> Vec<WorkspaceMember> {
|
||||
// Run `cargo metadata` command to get project metadata
|
||||
let output = Command::new("cargo")
|
||||
.arg("metadata")
|
||||
.output()
|
||||
.expect("Failed to execute command");
|
||||
|
||||
// Parse the JSON output
|
||||
let metadata: Value = serde_json::from_slice(&output.stdout).expect("Failed to parse JSON");
|
||||
|
||||
// Extract workspaces from the metadata, excluding examples/ and xtask
|
||||
let workspaces = metadata["workspace_members"]
|
||||
.as_array()
|
||||
.expect("Expected an array of workspace members")
|
||||
.iter()
|
||||
.filter_map(|member| {
|
||||
let parts: Vec<_> = member.as_str()?.split_whitespace().collect();
|
||||
let (workspace_name, workspace_path) =
|
||||
(parts.first()?.to_owned(), parts.last()?.to_owned());
|
||||
|
||||
let workspace_path = workspace_path.replace("(path+file://", "").replace(')', "");
|
||||
|
||||
match w_type {
|
||||
WorkspaceMemberType::Crate
|
||||
if workspace_name != "xtask" && !workspace_path.contains("examples/") =>
|
||||
{
|
||||
Some(WorkspaceMember::new(
|
||||
workspace_name.to_string(),
|
||||
workspace_path.to_string(),
|
||||
))
|
||||
}
|
||||
WorkspaceMemberType::Example
|
||||
if workspace_name != "xtask" && workspace_path.contains("examples/") =>
|
||||
{
|
||||
Some(WorkspaceMember::new(
|
||||
workspace_name.to_string(),
|
||||
workspace_path.to_string(),
|
||||
))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
workspaces
|
||||
}
|
||||
|
||||
/// Print duration as HH:MM:SS format
|
||||
pub(crate) fn format_duration(duration: &Duration) -> String {
|
||||
let seconds = duration.as_secs();
|
||||
let minutes = seconds / 60;
|
||||
let hours = minutes / 60;
|
||||
let remaining_minutes = minutes % 60;
|
||||
let remaining_seconds = seconds % 60;
|
||||
|
||||
format!(
|
||||
"{:02}:{:02}:{:02}",
|
||||
hours, remaining_minutes, remaining_seconds
|
||||
)
|
||||
}
|
||||
Loading…
Reference in New Issue