Build Fuchsia in CI

.github/workflows/ci.yml

@@ -288,6 +288,9 @@ jobs:
           - name: x86_64-gnu-aux
             os: ubuntu-20.04-4core-16gb
             env: {}
+          - name: x86_64-gnu-integration
+            os: ubuntu-20.04-16core-64gb
+            env: {}
           - name: x86_64-gnu-debug
             os: ubuntu-20.04-8core-32gb
             env: {}

src/bootstrap/src/core/build_steps/compile.rs

@@ -1722,7 +1722,7 @@ impl Step for Assemble {
             let dst_exe = exe("rust-lld", target_compiler.host);
             builder.copy(&lld_install.join("bin").join(&src_exe), &libdir_bin.join(&dst_exe));
             let self_contained_lld_dir = libdir_bin.join("gcc-ld");
-            t!(fs::create_dir(&self_contained_lld_dir));
+            t!(fs::create_dir_all(&self_contained_lld_dir));
             let lld_wrapper_exe = builder.ensure(crate::core::build_steps::tool::LldWrapper {
                 compiler: build_compiler,
                 target: target_compiler.host,

src/ci/docker/host-x86_64/dist-various-2/Dockerfile

@@ -76,8 +76,8 @@ RUN env \
     rm -rf /build/*
 
 WORKDIR /tmp
-COPY host-x86_64/dist-various-2/shared.sh /tmp/
+COPY scripts/shared.sh /tmp/
-COPY host-x86_64/dist-various-2/build-fuchsia-toolchain.sh /tmp/
+COPY scripts/build-fuchsia-toolchain.sh /tmp/
 RUN /tmp/build-fuchsia-toolchain.sh
 COPY host-x86_64/dist-various-2/build-solaris-toolchain.sh /tmp/
 RUN /tmp/build-solaris-toolchain.sh x86_64  amd64   solaris-i386  pc

src/ci/docker/host-x86_64/x86_64-gnu-integration/Dockerfile

@@ -0,0 +1,68 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  build-essential \
+  g++ \
+  make \
+  ninja-build \
+  file \
+  curl \
+  ca-certificates \
+  python3 \
+  git \
+  cmake \
+  libssl-dev \
+  sudo \
+  xz-utils \
+  pkg-config \
+  unzip \
+  && rm -rf /var/lib/apt/lists/*
+
+# Duplicated in dist-various-2 Dockerfile.
+# FIXME: Move to canonical triple
+ENV \
+    AR_x86_64_fuchsia=x86_64-unknown-fuchsia-ar \
+    CC_x86_64_fuchsia=x86_64-unknown-fuchsia-clang \
+    CFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \
+    CXX_x86_64_fuchsia=x86_64-unknown-fuchsia-clang++ \
+    CXXFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \
+    LDFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -L/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib"
+
+WORKDIR /tmp
+COPY scripts/shared.sh /tmp/
+COPY scripts/build-fuchsia-toolchain.sh /tmp/
+RUN /tmp/build-fuchsia-toolchain.sh
+
+ENV CARGO_TARGET_X86_64_FUCHSIA_AR /usr/local/bin/llvm-ar
+ENV CARGO_TARGET_X86_64_FUCHSIA_RUSTFLAGS \
+  -C panic=abort \
+  -C force-unwind-tables=yes \
+  -C link-arg=--sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot \
+  -Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot/lib \
+  -Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib
+
+ENV TARGETS=x86_64-fuchsia
+ENV TARGETS=$TARGETS,x86_64-unknown-linux-gnu
+
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
+
+ENV RUST_INSTALL_DIR /checkout/obj/install
+RUN mkdir -p $RUST_INSTALL_DIR/etc
+
+ENV RUST_CONFIGURE_ARGS \
+  --prefix=$RUST_INSTALL_DIR \
+  --sysconfdir=etc \
+  --enable-lld \
+  --llvm-libunwind=in-tree \
+  --enable-extended \
+  --disable-docs \
+  --set target.x86_64-fuchsia.cc=/usr/local/bin/clang \
+  --set target.x86_64-fuchsia.cxx=/usr/local/bin/clang++ \
+  --set target.x86_64-fuchsia.ar=/usr/local/bin/llvm-ar \
+  --set target.x86_64-fuchsia.ranlib=/usr/local/bin/llvm-ranlib \
+  --set target.x86_64-fuchsia.linker=/usr/local/bin/ld.lld
+ENV SCRIPT \
+    python3 ../x.py install --target $TARGETS compiler/rustc library/std clippy && \
+    bash ../src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh

src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# Downloads and builds the Fuchsia operating system using a toolchain installed
+# in $RUST_INSTALL_DIR.
+
+set -euf -o pipefail
+
+INTEGRATION_SHA=66793c4894bf6204579bbee3b79956335f31c768
+PICK_REFS=()
+
+checkout=fuchsia
+jiri=.jiri_root/bin/jiri
+
+set -x
+
+# This script will:
+# - create a directory named "fuchsia" if it does not exist
+# - download "jiri" to "fuchsia/.jiri_root/bin"
+curl -s "https://fuchsia.googlesource.com/jiri/+/HEAD/scripts/bootstrap_jiri?format=TEXT" \
+    | base64 --decode \
+    | bash -s $checkout
+
+cd $checkout
+
+$jiri init \
+    -partial=true \
+    -analytics-opt=false \
+    .
+
+$jiri import \
+    -name=integration \
+    -revision=$INTEGRATION_SHA \
+    -overwrite=true \
+    flower \
+    "https://fuchsia.googlesource.com/integration"
+
+if [ -d ".git" ]; then
+    # Wipe out any local changes if we're reusing a checkout.
+    git checkout --force JIRI_HEAD
+fi
+
+$jiri update -autoupdate=false
+
+echo integration commit = $(git -C integration rev-parse HEAD)
+
+for git_ref in "${PICK_REFS[@]}"; do
+    git fetch https://fuchsia.googlesource.com/fuchsia $git_ref
+    git cherry-pick --no-commit FETCH_HEAD
+done
+
+bash scripts/rust/build_fuchsia_from_rust_ci.sh

src/ci/docker/scripts/build-fuchsia-toolchain.sh

@@ -4,13 +4,13 @@ set -ex
 source shared.sh
 
 FUCHSIA_SDK_URL=https://chrome-infra-packages.appspot.com/dl/fuchsia/sdk/core/linux-amd64
-FUCHSIA_SDK_ID=4xjxrGUrDbQ6_zJwj6cDN1IbWsWV5aCQXC_zO_Hu0XkC
+FUCHSIA_SDK_ID=MrhQwtmP8CpZre-i_PNOREcThbUcrX3bA-45d6WQr-cC
-FUCHSIA_SDK_SHA256=e318f1ac652b0db43aff32708fa70337521b5ac595e5a0905c2ff33bf1eed179
+FUCHSIA_SDK_SHA256=32b850c2d98ff02a59adefa2fcf34e44471385b51cad7ddb03ee3977a590afe7
 FUCHSIA_SDK_USR_DIR=/usr/local/core-linux-amd64-fuchsia-sdk
 CLANG_DOWNLOAD_URL=\
 https://chrome-infra-packages.appspot.com/dl/fuchsia/third_party/clang/linux-amd64
-CLANG_DOWNLOAD_ID=vU0vNjSihOV4Q6taQYCpy03JXGiCyVwxen3rFMNMIgsC
+CLANG_DOWNLOAD_ID=Tpc85d1ZwSlZ6UKl2d96GRUBGNA5JKholOKe24sRDr0C
-CLANG_DOWNLOAD_SHA256=bd4d2f3634a284e57843ab5a4180a9cb4dc95c6882c95c317a7deb14c34c220b
+CLANG_DOWNLOAD_SHA256=4e973ce5dd59c12959e942a5d9df7a19150118d03924a86894e29edb8b110ebd
 
 install_clang() {
   mkdir -p clang_download

src/ci/docker/scripts/shared.sh

@@ -1,5 +1,11 @@
-#!/usr/bin/env bash
+#!/bin/false
-hide_output() {
+# shellcheck shell=bash
+
+# This file is intended to be sourced with `. shared.sh` or
+# `source shared.sh`, hence the invalid shebang and not being
+# marked as an executable file in git.
+
+function hide_output {
   { set +x; } 2>/dev/null
   on_err="
 echo ERROR: An error was encountered with the build.
@@ -15,7 +21,8 @@ exit 1
   set -x
 }
 
-# Copied from ../../shared.sh
+# See https://unix.stackexchange.com/questions/82598
+# Duplicated in src/ci/shared.sh
 function retry {
   echo "Attempting with retry:" "$@"
   local n=1

src/ci/github-actions/ci.yml

@@ -470,6 +470,9 @@ jobs:
           - name: x86_64-gnu-aux
             <<: *job-linux-4c
 
+          - name: x86_64-gnu-integration
+            <<: *job-linux-16c
+
           - name: x86_64-gnu-debug
             <<: *job-linux-8c
 
@@ -730,6 +733,7 @@ jobs:
               CODEGEN_BACKENDS: llvm,cranelift
             <<: *job-linux-16c
 
+
   master:
     name: master
     runs-on: ubuntu-latest

src/ci/shared.sh

@@ -8,7 +8,7 @@
 export MIRRORS_BASE="https://ci-mirrors.rust-lang.org/rustc"
 
 # See https://unix.stackexchange.com/questions/82598
-# Duplicated in docker/dist-various-2/shared.sh
+# Duplicated in docker/scripts/shared.sh
 function retry {
   echo "Attempting with retry:" "$@"
   local n=1