[libc++] Rework docker files for buildbots.

I've been sitting on this change for a while and have been using
it to build the bot images, so it should be upstream.

This re-configures the docker build files to use docker-compose
more heavily. This allows for composing large images with multiple
compilers without invalidating the docker caches.

After this commit I'll quickly switch all the current buildbots
over to a new docker image, followed by another update to add new
compilers

libcxx/utils/docker/debian9/Dockerfile.compiler_zoo

@@ -1,33 +0,0 @@
-#===- libcxx/utils/docker/debian9/Dockerfile --------------------------------------------------===//
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-#===-------------------------------------------------------------------------------------------===//
-
-#===-------------------------------------------------------------------------------------------===//
-#  compiler-zoo
-#===-------------------------------------------------------------------------------------------===//
-FROM  ericwf/libcxx-buildbot-base:latest AS compiler-zoo
-LABEL maintainer "libc++ Developers"
-
-# Copy over the GCC and Clang installations
-COPY --from=ericwf/compilers:latest /opt/gcc-4.8.5 /opt/gcc-4.8.5
-COPY --from=ericwf/compilers:latest /opt/gcc-4.9.4 /opt/gcc-4.9.4
-COPY --from=ericwf/compilers:latest /opt/gcc-5   /opt/gcc-5
-COPY --from=ericwf/compilers:latest /opt/gcc-6   /opt/gcc-6
-COPY --from=ericwf/compilers:latest /opt/gcc-7   /opt/gcc-7
-COPY --from=ericwf/compilers:latest /opt/gcc-8   /opt/gcc-8
-COPY --from=ericwf/compilers_tot:latest /opt/gcc-tot /opt/gcc-tot
-
-COPY --from=ericwf/compilers:latest /opt/llvm-3.6 /opt/llvm-3.6
-COPY --from=ericwf/compilers:latest /opt/llvm-3.7 /opt/llvm-3.7
-COPY --from=ericwf/compilers:latest /opt/llvm-3.8 /opt/llvm-3.8
-COPY --from=ericwf/compilers:latest /opt/llvm-3.9 /opt/llvm-3.9
-COPY --from=ericwf/compilers:latest /opt/llvm-4.0 /opt/llvm-4.0
-COPY --from=ericwf/compilers:latest /opt/llvm-5.0 /opt/llvm-5.0
-COPY --from=ericwf/compilers:latest /opt/llvm-6.0 /opt/llvm-6.0
-COPY --from=ericwf/compilers:latest /opt/llvm-7.0 /opt/llvm-7.0
-COPY --from=ericwf/compilers:latest /opt/llvm-8.0 /opt/llvm-8.0
-COPY --from=ericwf/compilers_tot:latest /opt/llvm-tot /opt/llvm-tot

libcxx/utils/docker/debian9/compilers/clang.Dockerfile

@@ -1,19 +1,18 @@
-#===- libcxx/utils/docker/debian9/Dockerfile --------------------------------------------------===//
+#===----------------------------------------------------------------------===//
 #
 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
-#===-------------------------------------------------------------------------------------------===//
+#===----------------------------------------------------------------------===//
 
 # Build GCC versions
-FROM ericwf/builder-base:latest
+FROM ericwf/llvm-builder-base:latest
 LABEL maintainer "libc++ Developers"
 
 ARG install_prefix
 ARG branch
 
-# Build additional LLVM versions
+ADD scripts/build_llvm_version.sh /tmp/
-
+RUN /tmp/build_llvm_version.sh --install "$install_prefix" --branch "$branch" \
-ADD scripts/build_llvm_version.sh /tmp/build_llvm_version.sh
+  && rm /tmp/build_llvm_version.sh
-RUN /tmp/build_llvm_version.sh --install "$install_prefix" --branch "$branch"

libcxx/utils/docker/debian9/compilers/compiler-zoo.Dockerfile

@@ -0,0 +1,35 @@
+#===- libcxx/utils/docker/debian9/Dockerfile --------------------------------------------------===//
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===-------------------------------------------------------------------------------------------===//
+
+#===-------------------------------------------------------------------------------------------===//
+#  compiler-zoo
+#===-------------------------------------------------------------------------------------------===//
+FROM  ericwf/llvm-builder-base:latest AS compiler-zoo
+LABEL maintainer "libc++ Developers"
+
+# Copy over the GCC and Clang installations
+COPY --from=ericwf/compiler:gcc-4.8.5 /opt/gcc-4.8.5 /opt/gcc-4.8.5
+COPY --from=ericwf/compiler:gcc-4.9.4 /opt/gcc-4.9.4 /opt/gcc-4.9.4
+COPY --from=ericwf/compiler:gcc-5 /opt/gcc-5   /opt/gcc-5
+COPY --from=ericwf/compiler:gcc-6 /opt/gcc-6   /opt/gcc-6
+COPY --from=ericwf/compiler:gcc-7 /opt/gcc-7   /opt/gcc-7
+COPY --from=ericwf/compiler:gcc-8 /opt/gcc-8   /opt/gcc-8
+COPY --from=ericwf/compiler:gcc-tot /opt/gcc-tot /opt/gcc-tot
+
+COPY --from=ericwf/compiler:llvm-3.6 /opt/llvm-3.6 /opt/llvm-3.6
+COPY --from=ericwf/compiler:llvm-3.7 /opt/llvm-3.7 /opt/llvm-3.7
+COPY --from=ericwf/compiler:llvm-3.8 /opt/llvm-3.8 /opt/llvm-3.8
+COPY --from=ericwf/compiler:llvm-3.9 /opt/llvm-3.9 /opt/llvm-3.9
+COPY --from=ericwf/compiler:llvm-4 /opt/llvm-4 /opt/llvm-4
+COPY --from=ericwf/compiler:llvm-5 /opt/llvm-5 /opt/llvm-5
+COPY --from=ericwf/compiler:llvm-6 /opt/llvm-6 /opt/llvm-6
+COPY --from=ericwf/compiler:llvm-7 /opt/llvm-7 /opt/llvm-7
+COPY --from=ericwf/compiler:llvm-8 /opt/llvm-8 /opt/llvm-8
+COPY --from=ericwf/compiler:llvm-tot /opt/llvm-tot /opt/llvm-tot
+
+

libcxx/utils/docker/debian9/compilers/compilers.yml

@@ -0,0 +1,156 @@
+version: '3.7'
+
+x-build-clang: &build-clang
+    context: .
+    dockerfile: compilers/clang.Dockerfile
+
+x-build-gcc: &build-gcc
+    context: .
+    dockerfile: compilers/gcc.Dockerfile
+
+services:
+  gcc-4.8.5:
+    build:
+      <<: *build-gcc
+      args:
+        branch: gcc-4_8_5-release
+        install_prefix: /opt/gcc-4.8.5
+        cherry_pick: ec1cc0263f156f70693a62cf17b254a0029f4852
+    image: ericwf/compiler:gcc-4.8.5
+  gcc-4.9.4:
+    build:
+      <<: *build-gcc
+      args:
+        branch: gcc-4_9_4-release
+        install_prefix: /opt/gcc-4.9.4
+    image: ericwf/compiler:gcc-4.9.4
+  gcc-5:
+    build:
+      <<: *build-gcc
+      args:
+        branch: gcc-5_5_0-release
+        install_prefix: /opt/gcc-5
+    image: ericwf/compiler:gcc-5
+  gcc-6:
+    build:
+      <<: *build-gcc
+      args:
+        branch: gcc-6_5_0-release
+        install_prefix: /opt/gcc-6
+    image: ericwf/compiler:gcc-6
+  gcc-7:
+    build:
+      <<: *build-gcc
+      args:
+        branch: gcc-7_4_0-release
+        install_prefix: /opt/gcc-7
+    image: ericwf/compiler:gcc-7
+  gcc-8:
+    build:
+      <<: *build-gcc
+      args:
+        branch: gcc-8_2_0-release
+        install_prefix: /opt/gcc-8
+    image: ericwf/compiler:gcc-8
+  # Add LLVM compilers
+  llvm-3.6:
+    build:
+      <<: *build-clang
+      args:
+        branch: release/3.6.x
+        install_prefix: /opt/llvm-3.6
+    image: ericwf/compiler:llvm-3.6
+  llvm-3.7:
+    build:
+      <<: *build-clang
+      args:
+        branch: release/3.7.x
+        install_prefix: /opt/llvm-3.7
+    image: ericwf/compiler:llvm-3.7
+  llvm-3.8:
+    build:
+      <<: *build-clang
+      args:
+        branch: release/3.8.x
+        install_prefix: /opt/llvm-3.8
+    image: ericwf/compiler:llvm-3.8
+  llvm-3.9:
+    build:
+      <<: *build-clang
+      args:
+        branch: release/3.9.x
+        install_prefix: /opt/llvm-3.9
+    image: ericwf/compiler:llvm-3.9
+  llvm-4:
+    build:
+      <<: *build-clang
+      args:
+        branch: release/4.x
+        install_prefix: /opt/llvm-4
+    image: ericwf/compiler:llvm-4
+  llvm-5:
+    build:
+      <<: *build-clang
+      args:
+        branch: release/5.x
+        install_prefix: /opt/llvm-5
+    image: ericwf/compiler:llvm-5
+  llvm-6:
+    build:
+      <<: *build-clang
+      args:
+        branch: release/6.x
+        install_prefix: /opt/llvm-6
+    image: ericwf/compiler:llvm-6
+  llvm-7:
+    build:
+      <<: *build-clang
+      args:
+        branch: release/7.x
+        install_prefix: /opt/llvm-7
+    image: ericwf/compiler:llvm-7
+  llvm-8:
+    build:
+      <<: *build-clang
+      args:
+        branch: release/8.x
+        install_prefix: /opt/llvm-8
+    image: ericwf/compiler:llvm-8
+  gcc-tot:
+    build:
+      <<: *build-gcc
+      args:
+        branch: master
+        install_prefix: /opt/gcc-tot
+    image: ericwf/compiler:gcc-tot
+  llvm-tot:
+    build:
+      <<: *build-clang
+      args:
+        branch: master
+        install_prefix: /opt/llvm-tot
+    image: ericwf/compiler:llvm-tot
+  compiler-zoo:
+    build:
+      context: .
+      dockerfile: compilers/compiler-zoo.Dockerfile
+      target: compiler-zoo
+    image: ericwf/compiler-zoo:latest
+    depends_on:
+      - gcc-4.8.5
+      - gcc-4.9.4
+      - gcc-5
+      - gcc-6
+      - gcc-7
+      - gcc-8
+      - gcc-tot
+      - llvm-3.6
+      - llvm-3.7
+      - llvm-3.8
+      - llvm-3.9
+      - llvm-4
+      - llvm-5
+      - llvm-6
+      - llvm-7
+      - llvm-8
+      - llvm-tot

libcxx/utils/docker/debian9/compilers/gcc.Dockerfile

@@ -7,12 +7,17 @@
 #===-------------------------------------------------------------------------------------------===//
 
 # Build GCC versions
-FROM ericwf/builder-base:latest
+FROM ericwf/llvm-builder-base:latest
 LABEL maintainer "libc++ Developers"
 
-ARG install_prefix
+
 ARG branch
 ARG cherry_pick=""
+ARG install_prefix
 
-ADD scripts/build_gcc_version.sh /tmp/build_gcc_version.sh
+ADD scripts/build_gcc_version.sh /tmp/
-RUN /tmp/build_gcc_version.sh --install "$install_prefix" --branch "$branch" --cherry-pick "$cherry_pick"
+RUN /tmp/build_gcc_version.sh \
+    --install "$install_prefix" \
+    --branch "$branch" \
+    --cherry-pick "$cherry_pick" \
+    && rm /tmp/build_gcc_version.sh

libcxx/utils/docker/debian9/docker-compose.yml

@@ -0,0 +1,14 @@
+version: '3.7'
+services:
+  llvm-builder-base:
+    build:
+      context: .
+      dockerfile: llvm-builder-base.Dockerfile
+      target: llvm-builder-base
+    image: ericwf/llvm-builder-base:new
+  llvm-buildbot-worker:
+    build:
+      context: .
+      dockerfile: llvm-buildbot-worker.Dockerfile
+      target: llvm-buildbot-worker
+    image: ericwf/llvm-buildbot-worker:new

libcxx/utils/docker/debian9/llvm-buildbot-worker.Dockerfile

@@ -2,25 +2,22 @@
 #===-------------------------------------------------------------------------------------------===//
 # buildslave
 #===-------------------------------------------------------------------------------------------===//
-FROM ericwf/builder-base:latest AS buildbot
+FROM ericwf/llvm-builder-base:latest AS llvm-buildbot-worker
 
-# Copy over the GCC and Clang installations
+COPY --from=ericwf/compiler:gcc-5 /opt/gcc-5 /opt/gcc-5
-COPY --from=ericwf/gcc-5:latest /opt/gcc-5 /opt/gcc-5
+COPY --from=ericwf/compiler:gcc-tot /opt/gcc-tot /opt/gcc-tot
-COPY --from=ericwf/gcc-tot:latest /opt/gcc-tot /opt/gcc-tot
+COPY --from=ericwf/compiler:llvm-4 /opt/llvm-4 /opt/llvm-4.0
-COPY --from=ericwf/llvm-4:latest /opt/llvm-4.0 /opt/llvm-4.0
 
+# FIXME(EricWF): Remove this hack once zorg has been updated.
 RUN ln -s /opt/gcc-5/bin/gcc /usr/local/bin/gcc-4.9 && \
     ln -s /opt/gcc-5/bin/g++ /usr/local/bin/g++-4.9
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-    bash-completion \
-    vim \
-    sudo \
     buildbot-slave \
   && rm -rf /var/lib/apt/lists/*
 
-ADD scripts/install_clang_packages.sh /tmp/install_clang_packages.sh
+ADD scripts/install_clang_packages.sh /tmp/
 RUN /tmp/install_clang_packages.sh && rm /tmp/install_clang_packages.sh
 
-RUN git clone https://git.llvm.org/git/libcxx.git /libcxx
+RUN rm -rf /llvm-project/ && git clone --depth=1 https://github.com/llvm/llvm-project.git /llvm-project

libcxx/utils/docker/debian9/llvm-builder-base.Dockerfile

@@ -1,13 +1,12 @@
-#===- libcxx/utils/docker/debian9/Dockerfile --------------------------------------------------===//
+#===----------------------------------------------------------------------===//
 #
 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
-#===-------------------------------------------------------------------------------------------===//
+#===----------------------------------------------------------------------===//
 
-# Setup the base builder image with the packages we'll need to build GCC and Clang from source.
+FROM launcher.gcr.io/google/debian9:latest AS llvm-builder-base
-FROM launcher.gcr.io/google/debian9:latest AS builder-base
 LABEL maintainer "libc++ Developers"
 
 RUN apt-get update && \
@@ -37,9 +36,12 @@ RUN apt-get update && \
       gnupg \
       apt-transport-https \
       sudo \
+      bash-completion \
+      vim \
       systemd \
       sysvinit-utils \
       systemd-sysv && \
   update-alternatives --install "/usr/bin/ld" "ld" "/usr/bin/ld.gold" 20 && \
   update-alternatives --install "/usr/bin/ld" "ld" "/usr/bin/ld.bfd" 10 && \
   rm -rf /var/lib/apt/lists/*
+

libcxx/utils/docker/debian9/scripts/buildbot/docker_start_buildbots.sh

@@ -3,6 +3,6 @@ set -x
 
 # Update the libc++ sources in the image in order to use the most recent version of
 # run_buildbots.sh
-cd /libcxx
+cd /llvm-project/
 git pull
-/libcxx/utils/docker/scripts/run_buildbot.sh "$@"
+/llvm-project/libcxx/utils/docker/debian9/scripts/buildbot/run_buildbot.sh "$@"

libcxx/utils/docker/debian9/scripts/buildbot/run_buildbot.sh

@@ -17,6 +17,13 @@ apt-get upgrade -y
 
 apt-get install sudo -y
 
+# FIXME(EricWF): Remove this hack. It's only in place to temporarily fix linking libclang_rt from the
+# debian packages.
+# WARNING: If you're not a buildbot, DO NOT RUN!
+apt-get install lld-10 -y
+rm /usr/bin/ld
+ln -s /usr/bin/lld-10 /usr/bin/ld
+
 systemctl set-property buildslave.service TasksMax=100000
 
 function setup_numbered_bot() {
@@ -100,3 +107,4 @@ done
 sleep 72000
 while pkill -SIGHUP buildslave; do sleep 5; done;
 shutdown now
+

libcxx/utils/docker/debian9/scripts/install_clang_packages.sh

@@ -20,7 +20,7 @@ Available options:
 EOF
 }
 
-VERSION="9"
+VERSION="10"
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
@@ -71,7 +71,7 @@ clang++ --version
 
 # Figure out the libc++ and libc++abi package versions that we want.
 if [ "$VERSION" == "" ]; then
-  VERSION="$(apt-cache search 'libc\+\+-[0-9]-dev' | awk '{print $1}' | awk -F- '{print $2}')"
+  VERSION="$(apt-cache search 'libc\+\+-[0-9]+-dev' | awk '{print $1}' | awk -F- '{print $2}')"
   echo "Installing version '$VERSION'"
 fi
 

libcxx/utils/docker/docker-compose.yml

@@ -1,46 +0,0 @@
-version: '3.4'
-services:
-  builder-base:
-    build:
-      context: .
-      dockerfile: debian9/Dockerfile.base
-      target: builder-base
-    image: ericwf/builder-base:latest
-  gcc-tot:
-    build:
-      context: .
-      dockerfile: debian9/Dockerfile.gcc
-      args:
-        install_prefix: /opt/gcc-tot
-        branch: master
-    image: ericwf/gcc-tot:latest
-  gcc-5:
-      build:
-        context: .
-        dockerfile: debian9/Dockerfile.gcc
-        args:
-          install_prefix: /opt/gcc-5
-          branch: gcc-5_5_0-release
-      image: ericwf/gcc-5:latest
-  llvm-4:
-    build:
-      context: .
-      dockerfile: debian9/Dockerfile.clang
-      args:
-        install_prefix: /opt/llvm-4.0
-        branch: release/4.x
-    image: ericwf/llvm-4:latest
-  llvm-tot:
-    build:
-      context: .
-      dockerfile: debian9/Dockerfile.clang
-      args:
-        install_prefix: /opt/llvm-tot
-        branch: master
-    image: ericwf/llvm-tot:latest
-  buildbot:
-    build:
-      context: .
-      dockerfile: debian9/Dockerfile.buildbot
-      target: buildbot
-    image: ericwf/libcxx-buildbot-base:latest

libcxx/utils/docker/scripts/docker_update_bot.sh

@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-
-set -x
-
-cd /libcxx
-git pull
-
-
-#pushd /tmp
-#curl -sSO https://dl.google.com/cloudagents/install-monitoring-agent.sh
-#bash install-monitoring-agent.sh
-#curl -sSO https://dl.google.com/cloudagents/install-logging-agent.sh
-#bash install-logging-agent.sh --structured
-#popd
-
-
-apt-get update -y
-apt-get upgrade -y
-
-apt-get install sudo -y
-
-systemctl set-property buildslave.service TasksMax=100000
-
-systemctl daemon-reload
-service buildslave restart