dnrops
/
smithy-rs
mirror of https://github.com/smithy-lang/smithy-rs
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix local CI on M1 Macs (#1346) 

The `acquire-build-image` script was coded up with the assumption that
it would only work with x86_64 build images. This commit revises that
script and the `Dockerfile` to correctly work on ARM64 architectures as
well.
This commit is contained in:
John DiSanti 2022-04-27 09:45:15 -07:00 committed by GitHub
parent ca849fb544
commit 0c011d635c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 85 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
tools/Dockerfile
View File

@ -13,7 +13,6 @@ FROM ${base_image} AS bare_base_image
#
FROM bare_base_image AS install_node
ARG node_version=v16.14.0
ARG node_bundle_sha256=0570b9354959f651b814e56a4ce98d4a067bf2385b9a0e6be075739bc65b0fae
ENV DEST_PATH=/opt/nodejs \
PATH=/opt/nodejs/bin:${PATH}
RUN yum -y updateinfo && \
@ -25,12 +24,20 @@ RUN yum -y updateinfo && \
yum clean all
WORKDIR /root
RUN set -eux; \
curl https://nodejs.org/dist/${node_version}/node-${node_version}-linux-x64.tar.xz --output node.tar.xz; \
echo "${node_bundle_sha256} node.tar.xz" | sha256sum --check; \
ARCHITECTURE=""; \
if [[ "$(uname -m)" == "aarch64" || "$(uname -m)" == "arm64" ]]; then \
curl "https://nodejs.org/dist/${node_version}/node-${node_version}-linux-arm64.tar.xz" --output node.tar.xz; \
echo "5a6e818c302527a4b1cdf61d3188408c8a3e4a1bbca1e3f836c93ea8469826ce node.tar.xz" | sha256sum --check; \
ARCHITECTURE="arm64"; \
else \
curl "https://nodejs.org/dist/${node_version}/node-${node_version}-linux-x64.tar.xz" --output node.tar.xz; \
echo "0570b9354959f651b814e56a4ce98d4a067bf2385b9a0e6be075739bc65b0fae node.tar.xz" | sha256sum --check; \
ARCHITECTURE="x64"; \
fi; \
mkdir -p "${DEST_PATH}"; \
tar -xJvf node.tar.xz -C "${DEST_PATH}"; \
mv "${DEST_PATH}/node-${node_version}-linux-x64/"* "${DEST_PATH}"; \
rmdir "${DEST_PATH}"/node-${node_version}-linux-x64; \
mv "${DEST_PATH}/node-${node_version}-linux-${ARCHITECTURE}/"* "${DEST_PATH}"; \
rmdir "${DEST_PATH}"/node-${node_version}-linux-${ARCHITECTURE}; \
rm node.tar.xz; \
node --version
@ -65,8 +72,13 @@ RUN yum -y updateinfo && \
pkgconfig && \
yum clean all
RUN set -eux; \
curl https://static.rust-lang.org/rustup/archive/1.24.3/x86_64-unknown-linux-gnu/rustup-init --output rustup-init; \
echo "3dc5ef50861ee18657f9db2eeb7392f9c2a6c95c90ab41e45ab4ca71476b4338 rustup-init" | sha256sum --check; \
if [[ "$(uname -m)" == "aarch64" || "$(uname -m)" == "arm64" ]]; then \
curl https://static.rust-lang.org/rustup/archive/1.24.3/aarch64-unknown-linux-gnu/rustup-init --output rustup-init; \
echo "32a1532f7cef072a667bac53f1a5542c99666c4071af0c9549795bbdb2069ec1 rustup-init" | sha256sum --check; \
else \
curl https://static.rust-lang.org/rustup/archive/1.24.3/x86_64-unknown-linux-gnu/rustup-init --output rustup-init; \
echo "3dc5ef50861ee18657f9db2eeb7392f9c2a6c95c90ab41e45ab4ca71476b4338 rustup-init" | sha256sum --check; \
fi; \
chmod +x rustup-init; \
./rustup-init -y --no-modify-path --profile minimal --default-toolchain ${rust_stable_version}; \
rm rustup-init; \
@ -121,7 +133,7 @@ COPY --chown=build:build --from=install_rust /opt/rustup /opt/rustup
ENV PATH=/opt/cargo/bin:/opt/nodejs/bin:$PATH \
CARGO_HOME=/opt/cargo \
RUSTUP_HOME=/opt/rustup \
JAVA_HOME=/usr/lib/jvm/java-11-amazon-corretto.x86_64 \
JAVA_HOME=/usr/lib/jvm/jre-11-openjdk \
GRADLE_USER_HOME=/home/build/.gradle \
RUST_STABLE_VERSION=${rust_stable_version} \
RUST_NIGHTLY_VERSION=${rust_nightly_version} \

71
tools/ci-build/acquire-build-image
View File

@ -27,10 +27,16 @@ def announce(message):
class DockerPullResult(Enum):
SUCCESS = 1
ERROR_THROTTLED = 2
RETRYABLE_ERROR = 3
NOT_FOUND = 4
UNKNOWN_ERROR = 5
REMOTE_ARCHITECTURE_MISMATCH = 2
ERROR_THROTTLED = 3
RETRYABLE_ERROR = 4
NOT_FOUND = 5
UNKNOWN_ERROR = 6
class Platform(Enum):
X86_64 = 0
ARM_64 = 1
# Script context
@ -65,6 +71,13 @@ class Context:
# Mockable shell commands
class Shell:
# Returns the platform that this script is running on
def platform(self):
(_, stdout, _) = get_cmd_output("uname -m")
if stdout == "arm64":
return Platform.ARM_64
return Platform.X86_64
# Returns True if the given `image_name` and `image_tag` exist locally
def docker_image_exists_locally(self, image_name, image_tag):
(status, _, _) = get_cmd_output(f"docker inspect \"{image_name}:{image_tag}\"", check=False)
@ -117,6 +130,8 @@ class Shell:
# Pulls a Docker image and retries if it gets throttled
def docker_pull_with_retry(shell, image_name, image_tag, throttle_sleep_time=45, retryable_error_sleep_time=1):
if shell.platform() == Platform.ARM_64:
return DockerPullResult.REMOTE_ARCHITECTURE_MISMATCH
for attempt in range(1, 5):
announce(f"Attempting to pull remote image {image_name}:{image_tag} (attempt {attempt})...")
result = shell.docker_pull(image_name, image_tag)
@ -155,15 +170,19 @@ def acquire_build_image(context=Context.default(), shell=Shell()):
announce("Base image not found locally.")
pull_result = docker_pull_with_retry(shell, REMOTE_BASE_IMAGE_NAME, context.image_tag)
if pull_result != DockerPullResult.SUCCESS:
if pull_result == DockerPullResult.UNKNOWN_ERROR:
if pull_result == DockerPullResult.REMOTE_ARCHITECTURE_MISMATCH:
announce("Remote architecture is not the same as the local architecture. A local build is required.")
elif pull_result == DockerPullResult.UNKNOWN_ERROR:
announce("An unknown failure happened during Docker pull. This needs to be examined.")
return 1
else:
announce("Failed to pull remote image, which can happen if it doesn't exist.")
if not context.allow_local_build:
announce("Local build turned off by ALLOW_LOCAL_BUILD env var. Aborting.")
return 1
announce("Failed to pull remote image, which can happen if it doesn't exist. Building a new image locally.")
announce("Building a new image locally.")
shell.docker_build_base_image(context.image_tag, context.tools_path)
if context.github_actions:
@ -199,6 +218,7 @@ class SelfTest(unittest.TestCase):
def mock_shell(self):
shell = Shell()
shell.platform = MagicMock()
shell.docker_build_base_image = MagicMock()
shell.docker_build_build_image = MagicMock()
shell.docker_image_exists_locally = MagicMock()
@ -207,6 +227,20 @@ class SelfTest(unittest.TestCase):
shell.docker_tag = MagicMock()
return shell
def test_retry_architecture_mismatch(self):
shell = self.mock_shell()
shell.platform.side_effect = [Platform.ARM_64]
self.assertEqual(
DockerPullResult.REMOTE_ARCHITECTURE_MISMATCH,
docker_pull_with_retry(
shell,
"test-image",
"test-image-tag",
throttle_sleep_time=0,
retryable_error_sleep_time=0
)
)
def test_retry_immediate_success(self):
shell = self.mock_shell()
shell.docker_pull.side_effect = [DockerPullResult.SUCCESS]
@ -327,6 +361,7 @@ class SelfTest(unittest.TestCase):
# It should: build a local build image using that local base image
def test_image_exists_locally_already(self):
shell = self.mock_shell()
shell.platform.side_effect = [Platform.X86_64]
shell.docker_image_exists_locally.side_effect = [True]
self.assertEqual(0, acquire_build_image(self.test_context(), shell))
@ -344,6 +379,7 @@ class SelfTest(unittest.TestCase):
def test_image_local_build(self):
context = self.test_context(allow_local_build=True)
shell = self.mock_shell()
shell.platform.side_effect = [Platform.X86_64]
shell.docker_image_exists_locally.side_effect = [False]
shell.docker_pull.side_effect = [DockerPullResult.NOT_FOUND]
@ -354,6 +390,26 @@ class SelfTest(unittest.TestCase):
shell.docker_tag.assert_called_with(LOCAL_BASE_IMAGE_NAME, "someimagetag", LOCAL_BASE_IMAGE_NAME, LOCAL_TAG)
shell.docker_build_build_image.assert_called_with("123", "/tmp/test/script-path")
# When:
# - the base image doesn't exist locally
# - the base image exists remotely
# - local builds are allowed
# - there is a difference in platform between local and remote
# - NOT running in GitHub Actions
# It should: build a local image from scratch and NOT save it to file
def test_image_local_build_architecture_mismatch(self):
context = self.test_context(allow_local_build=True)
shell = self.mock_shell()
shell.platform.side_effect = [Platform.ARM_64]
shell.docker_image_exists_locally.side_effect = [False]
self.assertEqual(0, acquire_build_image(context, shell))
shell.docker_image_exists_locally.assert_called_once()
shell.docker_build_base_image.assert_called_with("someimagetag", "/tmp/test/tools-path")
shell.docker_save.assert_not_called()
shell.docker_tag.assert_called_with(LOCAL_BASE_IMAGE_NAME, "someimagetag", LOCAL_BASE_IMAGE_NAME, LOCAL_TAG)
shell.docker_build_build_image.assert_called_with("123", "/tmp/test/script-path")
# When:
# - the base image doesn't exist locally
# - the base image doesn't exist remotely
@ -363,6 +419,7 @@ class SelfTest(unittest.TestCase):
def test_image_local_build_github_actions(self):
context = self.test_context(allow_local_build=True, github_actions=True)
shell = self.mock_shell()
shell.platform.side_effect = [Platform.X86_64]
shell.docker_image_exists_locally.side_effect = [False]
shell.docker_pull.side_effect = [DockerPullResult.NOT_FOUND]
@ -385,6 +442,7 @@ class SelfTest(unittest.TestCase):
def test_image_fail_local_build_disabled(self):
context = self.test_context(allow_local_build=False)
shell = self.mock_shell()
shell.platform.side_effect = [Platform.X86_64]
shell.docker_image_exists_locally.side_effect = [False]
shell.docker_pull.side_effect = [DockerPullResult.NOT_FOUND]
@ -402,6 +460,7 @@ class SelfTest(unittest.TestCase):
def test_pull_remote_image(self):
context = self.test_context(allow_local_build=False)
shell = self.mock_shell()
shell.platform.side_effect = [Platform.X86_64]
shell.docker_image_exists_locally.side_effect = [False]
shell.docker_pull.side_effect = [DockerPullResult.SUCCESS]