From a6c3aba5eb01a35305995c46eeb46ceda3cad124 Mon Sep 17 00:00:00 2001 From: Thomas Cameron Date: Thu, 8 Jun 2023 00:16:09 +0900 Subject: [PATCH] RFC30: Compile time benchmark (#2617) ## Motivation and Context This PR implements ci-script for bench marking compile time. Results are turned into a markdown file and compile time is normalized to a value relative to the compile time of S3's sdk on dev profile without any features. Benchmark is triggered for every PR. I considered using AWS batch, however, I decided not to move forward with for following reasons - I do not have access to your AWS account, thus making the debugging extremely difficult - Fork's github action will always fail if you block access ## Testing NA ## Checklist - [ ] I have updated `CHANGELOG.next.toml` if I made changes to the smithy-rs codegen or runtime crates - [ ] I have updated `CHANGELOG.next.toml` if I made changes to the AWS SDK, generated SDK code, or SDK runtime crates ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --------- Co-authored-by: Zelda Hessler --- .github/workflows/pull-request-bot.yml | 39 ++++++++++++++++++++++++- tools/ci-scripts/compiletime-benchmark | 33 +++++++++++++++++++++ tools/compiletime-benchmark/format.py | 40 ++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 1 deletion(-) create mode 100755 tools/ci-scripts/compiletime-benchmark create mode 100644 tools/compiletime-benchmark/format.py diff --git a/.github/workflows/pull-request-bot.yml b/.github/workflows/pull-request-bot.yml index aafeaf773..e07a79fe9 100644 --- a/.github/workflows/pull-request-bot.yml +++ b/.github/workflows/pull-request-bot.yml @@ -135,10 +135,45 @@ jobs: run: | aws s3 cp target/doc "s3://${S3_BUCKET_NAME}/docs/${{ inputs.head_revision }}" --recursive + compiletime-benchmark: + runs-on: ubuntu-latest + name: Run Compiletime Benchmark + permissions: + id-token: write + contents: read + pull-requests: write + outputs: + compiletime-benchmark: ${{ steps.compiletime-benchmark.outputs.compiletime-benchmark }} + steps: + - uses: actions/checkout@v3 + - uses: actions/cache@v3 + name: Gradle Cache + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} + restore-keys: | + ${{ runner.os }}-gradle- + # JDK is needed to generate code + - name: Set up JDK + uses: actions/setup-java@v3 + with: + distribution: corretto + java-package: jdk + java-version: ${{ env.java_version }} + - uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ env.rust_version }} + - name: run benchmark + id: run-compiletime-benchmark + run: bash tools/ci-scripts/compiletime-benchmark && cat /tmp/compiletime-benchmark.md >> "$GITHUB_OUTPUT" + post-bot-comment: needs: - generate-diff - generate-doc-preview + - compiletime-benchmark runs-on: ubuntu-latest name: Post bot comment permissions: @@ -164,6 +199,8 @@ jobs: issue_number: ${{ inputs.issue_number }}, owner: context.repo.owner, repo: context.repo.repo, - body: '${{ steps.bot-messages.outputs.codegen-diff }}\n\n' + + body: '${{ steps.compiletime-benchmark.outputs.compiletime-benchmark }}\n\n' + + '${{ steps.bot-messages.outputs.codegen-diff }}\n\n' + '${{ needs.generate-doc-preview.outputs.bot-message }}\n\n' + }) diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark new file mode 100755 index 000000000..3b0a18cc7 --- /dev/null +++ b/tools/ci-scripts/compiletime-benchmark @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# +set -eux + +export TIMEFORMAT=%R + +function compile() { + cd $1 && + export CARGORUSTFLAG="" && + cargo build &>/dev/null && cargo clean && # this is for downloading crates + \time --format="%e seconds" bash -c "cargo build &> /dev/null" && cargo clean && + \time --format="%e seconds" bash -c "cargo build --release &> /dev/null" && cargo clean && + export CARGORUSTFLAG="--cfg aws_sdk_unstable" && + \time --format="%e seconds" bash -c "cargo build --all-features &>/dev/null" && cargo clean && + \time --format="%e seconds" bash -c "cargo build --all-features --release &>/dev/null" && cargo clean +} + +./gradlew :aws:sdk:assemble +DIR=$PWD +for variable in $(dir "aws/sdk/build/aws-sdk/sdk"); do + if [[ $variable != *"aws-"* ]]; then + echo "START" &>>/tmp/compiletime-benchmark.txt + echo "$variable" &>>/tmp/compiletime-benchmark.txt + compile "$DIR/aws/sdk/build/aws-sdk/sdk/$variable" &>>/tmp/compiletime-benchmark.txt + echo "END" &>>/tmp/compiletime-benchmark.txt + fi +done + +cd $DIR +python3 tools/compiletime-benchmark/format.py diff --git a/tools/compiletime-benchmark/format.py b/tools/compiletime-benchmark/format.py new file mode 100644 index 000000000..f74e06bd1 --- /dev/null +++ b/tools/compiletime-benchmark/format.py @@ -0,0 +1,40 @@ +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# + +import itertools + + +def main(): + f = open("/tmp/compiletime-benchmark.txt", "r").read() + iter = map(lambda x: x.split("END"), f.split("START")) + iter = itertools.chain.from_iterable(iter) + markdown = """ + | sdk name | dev | release | dev all features | release all features | + | -------- | --- | ------- | ---------------- | -------------------- | + """ + idx = 0 + for i in iter: + idx += 1 + print("============") + print(idx) + print(i) + print("============") + + lines = i.splitlines() + if len(lines) > 1: + continue + + sdk_name = lines[0] + row = "\n|" + sdk_name + \ + "|".join(map(lambda x: float(x), lines[1:])) + "|" + + markdown += row + + print(markdown) + with open("/tmp/compiletime-benchmark.md", "w") as f: + f.write(markdown) + f.flush() + +main()