mirror of https://github.com/rust-lang/rust.git
Auto merge of #123451 - Kobzol:arbitrary-try-build, r=pietroalbini
CI: add a script for dynamically computing CI job matrix It would be great if was easier to run specific CI workflows locally, and also to allow us to spawn a specific CI workflow by bors, to enable running arbitrary try builds. See discussion [here](https://rust-lang.zulipchat.com/#narrow/stream/242791-t-infra/topic/CI.20workflows.20refactoring). This PR is a first step in that direction. - Moves the definition of CI runners and (for now) PR jobs into a separate `jobs.yml` file. - Adds a simple Python script that reads the file, decides which jobs should be active for the current CI workflow, and prints them as JSON to their output. - The PR job then reads this output and generates its job matrix based on it. By moving the job definitions from `ci.yml` into a separate file, we can handle it programmatically, which should make it easier to both do local execution of CI jobs and also to do arbitrary try builds.
This commit is contained in:
bors
commit
ccfcd950b3
|
|
@ -36,8 +36,21 @@ concurrency:
|
|||
group: "${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }}"
|
||||
cancel-in-progress: true
|
||||
jobs:
|
||||
calculate_matrix:
|
||||
name: Calculate job matrix
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
jobs: "${{ steps.jobs.outputs.jobs }}"
|
||||
steps:
|
||||
- name: Checkout the source code
|
||||
uses: actions/checkout@v4
|
||||
- name: Calculate the CI job matrix
|
||||
run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT
|
||||
id: jobs
|
||||
pr:
|
||||
name: "PR - ${{ matrix.name }}"
|
||||
needs:
|
||||
- calculate_matrix
|
||||
env:
|
||||
PR_CI_JOB: 1
|
||||
CI_JOB_NAME: "${{ matrix.name }}"
|
||||
|
|
@ -51,20 +64,7 @@ jobs:
|
|||
continue-on-error: "${{ matrix.name == 'mingw-check-tidy' }}"
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- name: mingw-check
|
||||
os: ubuntu-20.04-4core-16gb
|
||||
env: {}
|
||||
- name: mingw-check-tidy
|
||||
os: ubuntu-20.04-4core-16gb
|
||||
env: {}
|
||||
- name: x86_64-gnu-llvm-17
|
||||
env:
|
||||
ENABLE_GCC_CODEGEN: "1"
|
||||
os: ubuntu-20.04-16core-64gb
|
||||
- name: x86_64-gnu-tools
|
||||
os: ubuntu-20.04-16core-64gb
|
||||
env: {}
|
||||
include: "${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}"
|
||||
defaults:
|
||||
run:
|
||||
shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}"
|
||||
|
|
|
|||
|
|
@ -341,9 +341,22 @@ concurrency:
|
|||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml.
|
||||
calculate_matrix:
|
||||
name: Calculate job matrix
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
jobs: ${{ steps.jobs.outputs.jobs }}
|
||||
steps:
|
||||
- name: Checkout the source code
|
||||
uses: actions/checkout@v4
|
||||
- name: Calculate the CI job matrix
|
||||
run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT
|
||||
id: jobs
|
||||
pr:
|
||||
<<: *base-ci-job
|
||||
name: PR - ${{ matrix.name }}
|
||||
needs: [ calculate_matrix ]
|
||||
env:
|
||||
<<: [*shared-ci-variables, *public-variables]
|
||||
PR_CI_JOB: 1
|
||||
|
|
@ -351,20 +364,8 @@ jobs:
|
|||
continue-on-error: ${{ matrix.name == 'mingw-check-tidy' }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- name: mingw-check
|
||||
<<: *job-linux-4c
|
||||
|
||||
- name: mingw-check-tidy
|
||||
<<: *job-linux-4c
|
||||
|
||||
- name: x86_64-gnu-llvm-17
|
||||
env:
|
||||
ENABLE_GCC_CODEGEN: "1"
|
||||
<<: *job-linux-16c
|
||||
|
||||
- name: x86_64-gnu-tools
|
||||
<<: *job-linux-16c
|
||||
# Check the `calculate_matrix` job to see how is the matrix defined.
|
||||
include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}
|
||||
|
||||
auto:
|
||||
<<: *base-ci-job
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
# This file contains definitions of CI job parameters that are loaded
|
||||
# dynamically in CI from ci.yml.
|
||||
# You *do not* need to re-run `src/tools/expand-yaml-anchors` when you
|
||||
# modify this file.
|
||||
shared_defs:
|
||||
- &base-job
|
||||
env: { }
|
||||
|
||||
- &job-linux-4c
|
||||
os: ubuntu-20.04-4core-16gb
|
||||
<<: *base-job
|
||||
|
||||
- &job-linux-8c
|
||||
os: ubuntu-20.04-8core-32gb
|
||||
<<: *base-job
|
||||
|
||||
- &job-linux-16c
|
||||
os: ubuntu-20.04-16core-64gb
|
||||
<<: *base-job
|
||||
|
||||
- &job-macos-xl
|
||||
os: macos-13 # We use the standard runner for now
|
||||
<<: *base-job
|
||||
|
||||
- &job-macos-m1
|
||||
os: macos-14
|
||||
<<: *base-job
|
||||
|
||||
- &job-windows-8c
|
||||
os: windows-2019-8core-32gb
|
||||
<<: *base-job
|
||||
|
||||
- &job-windows-16c
|
||||
os: windows-2019-16core-64gb
|
||||
<<: *base-job
|
||||
|
||||
- &job-aarch64-linux
|
||||
os: [ self-hosted, ARM64, linux ]
|
||||
|
||||
pr:
|
||||
- name: mingw-check
|
||||
<<: *job-linux-4c
|
||||
- name: mingw-check-tidy
|
||||
<<: *job-linux-4c
|
||||
- name: x86_64-gnu-llvm-17
|
||||
env:
|
||||
ENABLE_GCC_CODEGEN: "1"
|
||||
<<: *job-linux-16c
|
||||
- name: x86_64-gnu-tools
|
||||
<<: *job-linux-16c
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
This script serves for generating a matrix of jobs that should
|
||||
be executed on CI.
|
||||
|
||||
It reads job definitions from `src/ci/github-actions/jobs.yml`
|
||||
and filters them based on the event that happened on CI.
|
||||
|
||||
Currently, it only supports PR builds.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
JOBS_YAML_PATH = Path(__file__).absolute().parent.parent / "github-actions" / "jobs.yml"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
with open(JOBS_YAML_PATH) as f:
|
||||
jobs = yaml.safe_load(f)
|
||||
job_output = jobs["pr"]
|
||||
print(f"jobs={json.dumps(job_output)}")
|
||||
|
|
@ -2,11 +2,11 @@ use std::error::Error;
|
|||
use std::path::{Path, PathBuf};
|
||||
use yaml_rust::{Yaml, YamlEmitter, YamlLoader};
|
||||
|
||||
/// List of directories containing files to expand. The first tuple element is the source
|
||||
/// directory, while the second tuple element is the destination directory.
|
||||
/// List of files to expand. The first tuple element is the source
|
||||
/// file, while the second tuple element is the destination file.
|
||||
#[rustfmt::skip]
|
||||
static TO_EXPAND: &[(&str, &str)] = &[
|
||||
("src/ci/github-actions", ".github/workflows"),
|
||||
("src/ci/github-actions/ci.yml", ".github/workflows/ci.yml"),
|
||||
];
|
||||
|
||||
/// Name of a special key that will be removed from all the maps in expanded configuration files.
|
||||
|
|
@ -62,27 +62,20 @@ impl App {
|
|||
fn run(&self) -> Result<(), Box<dyn Error>> {
|
||||
for (source, dest) in TO_EXPAND {
|
||||
let source = self.base.join(source);
|
||||
let dest = self.base.join(dest);
|
||||
for entry in std::fs::read_dir(&source)? {
|
||||
let path = entry?.path();
|
||||
if !path.is_file() || path.extension().and_then(|e| e.to_str()) != Some("yml") {
|
||||
continue;
|
||||
}
|
||||
let dest_path = self.base.join(dest);
|
||||
|
||||
let dest_path = dest.join(path.file_name().unwrap());
|
||||
self.expand(&path, &dest_path).with_context(|| match self.mode {
|
||||
Mode::Generate => format!(
|
||||
"failed to expand {} into {}",
|
||||
self.path(&path),
|
||||
self.path(&dest_path)
|
||||
),
|
||||
Mode::Check => format!(
|
||||
"{} is not up to date; please run \
|
||||
self.expand(&source, &dest_path).with_context(|| match self.mode {
|
||||
Mode::Generate => format!(
|
||||
"failed to expand {} into {}",
|
||||
self.path(&source),
|
||||
self.path(&dest_path)
|
||||
),
|
||||
Mode::Check => format!(
|
||||
"{} is not up to date; please run \
|
||||
`x.py run src/tools/expand-yaml-anchors`.",
|
||||
self.path(&dest_path)
|
||||
),
|
||||
})?;
|
||||
}
|
||||
self.path(&dest_path)
|
||||
),
|
||||
})?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue