Build and publish multi-arch images and manifest (#3586)

* build: use HOST_ARCHITECTURE in bazel-generate.sh Use HOST_ARCHITECTURE for generating bazel config. There is no need to do cross compiling in this process. Signed-off-by: howard zhang <howard.zhang@arm.com> * mulitarch build: migrate multi-arch build script To sync the multi-arch build process with kubevirt/kubevirt project, migrate two main multi-arch build scripts from kubevirt/kubevirt project. kubevirt/kubevirt/hack/multi-arch.sh kubevirt/kubevirt/hack/push-container-manifest.sh Signed-off-by: howard zhang <howard.zhang@arm.com> * multiarch build: support build and push multiarch image and manifest Signed-off-by: howard zhang <howard.zhang@arm.com> * multiarch: setup multi-arch build process in automation Signed-off-by: howard zhang <howard.zhang@arm.com> --------- Signed-off-by: howard zhang <howard.zhang@arm.com>
2025-06-03 06:30:22 +00:00 · 2025-01-21 23:12:50 +08:00 · 2025-01-21 23:12:50 +08:00 · fcd5245001
commit fcd5245001
parent 27dc66bb7b
8 changed files with 183 additions and 26 deletions
--- a/11
+++ b/11
@ -165,19 +165,20 @@ cluster-sync: cluster-sync-cdi cluster-sync-test-infra ## Build the controller/i

 ##@ Bazel
 bazel-generate: ## Generate BUILD files for Bazel.
-	${DO_BAZ} "BUILD_ARCH=${BUILD_ARCH} ./hack/build/bazel-generate.sh -- staging/src pkg/ tools/ tests/ cmd/ vendor/"
+	${DO_BAZ} "./hack/build/bazel-generate.sh -- staging/src pkg/ tools/ tests/ cmd/ vendor/"

 bazel-cdi-generate:
-	${DO_BAZ} "BUILD_ARCH=${BUILD_ARCH} ./hack/build/bazel-generate.sh -- staging/src pkg/ tools/ tests/ cmd/"
+	${DO_BAZ} "./hack/build/bazel-generate.sh -- staging/src pkg/ tools/ tests/ cmd/"

 bazel-build: ## Build all Go binaries.
-	${DO_BAZ} "BUILD_ARCH=${BUILD_ARCH} ./hack/build/bazel-build.sh"
+	${DO_BAZ} "BUILD_ARCH=${BUILD_ARCH} ./hack/build/multi-arch.sh build"

 bazel-build-images:	bazel-cdi-generate bazel-build ## Build all the container images used (for both CDI and functional tests)
-	${DO_BAZ} "BUILD_ARCH=${BUILD_ARCH} DOCKER_PREFIX=${DOCKER_PREFIX} DOCKER_TAG=${DOCKER_TAG} ./hack/build/bazel-build-images.sh"
+	${DO_BAZ} "BUILD_ARCH=${BUILD_ARCH} DOCKER_PREFIX=${DOCKER_PREFIX} DOCKER_TAG=${DOCKER_TAG} ./hack/build/multi-arch.sh build-images"

 bazel-push-images: bazel-cdi-generate bazel-build ## Push the built container images to the registry defined in DOCKER_PREFIX
-	${DO_BAZ} "BUILD_ARCH=${BUILD_ARCH} DOCKER_PREFIX=${DOCKER_PREFIX} DOCKER_TAG=${DOCKER_TAG} DOCKER_CA_CERT_FILE=${DOCKER_CA_CERT_FILE} ./hack/build/bazel-push-images.sh"
+	${DO_BAZ} "BUILD_ARCH=${BUILD_ARCH} DOCKER_PREFIX=${DOCKER_PREFIX} DOCKER_TAG=${DOCKER_TAG} DOCKER_CA_CERT_FILE=${DOCKER_CA_CERT_FILE} ./hack/build/multi-arch.sh push-images"
+	BUILD_ARCH=${BUILD_ARCH} DOCKER_PREFIX=${DOCKER_PREFIX} DOCKER_TAG=${DOCKER_TAG} hack/build/push-container-manifest.sh

 push: bazel-push-images ## Same as bazel-push-images

--- a/automation/prow_periodic_push.sh
+++ b/automation/prow_periodic_push.sh
@ -3,30 +3,19 @@ set -e
 build_date="$(date +%Y%m%d)"
 cat "$QUAY_PASSWORD" | docker login --username $(cat "$QUAY_USER") --password-stdin=true quay.io

-case $BUILD_ARCH in
-  crossbuild-s390x|s390x)
-    ARCH_SUFFIX="-s390x"
-    ;;
-  crossbuild-aarch64|aarch64)
-    ARCH_SUFFIX="-arm64"
-    ;;
-  *)
-    echo "No BUILD_ARCH or ${BUILD_ARCH}, assuming amd64"
-    ;;
-esac
-
-export DOCKER_TAG="${build_date}_$(git show -s --format=%h)${ARCH_SUFFIX}"
+export DOCKER_TAG="${build_date}_$(git show -s --format=%h)"
+export BUILD_ARCH=s390x,aarch64,x86_64

 make manifests
 make bazel-push-images

 base_url="kubevirt-prow/devel/nightly/release/kubevirt/containerized-data-importer"
 bucket_dir="${base_url}/${build_date}"
-gsutil cp ./_out/manifests/release/cdi-operator.yaml gs://$bucket_dir/cdi-operator${ARCH_SUFFIX}.yaml
-gsutil cp ./_out/manifests/release/cdi-cr.yaml gs://$bucket_dir/cdi-cr${ARCH_SUFFIX}.yaml
+gsutil cp ./_out/manifests/release/cdi-operator.yaml gs://$bucket_dir/cdi-operator.yaml
+gsutil cp ./_out/manifests/release/cdi-cr.yaml gs://$bucket_dir/cdi-cr.yaml

 echo ${build_date} > ./_out/build_date
-gsutil cp ./_out/build_date gs://${base_url}/latest${ARCH_SUFFIX}
+gsutil cp ./_out/build_date gs://${base_url}/latest

 git show -s --format=%H > ./_out/commit
-gsutil cp ./_out/commit gs://${bucket_dir}/commit${ARCH_SUFFIX}
+gsutil cp ./_out/commit gs://${bucket_dir}/commit
--- a/hack/build/bazel-build-images.sh
+++ b/hack/build/bazel-build-images.sh
@ -33,3 +33,12 @@ for tag in ${docker_tag}; do
        --host_force_python=PY3 \
        //:test-container-images //cmd/cdi-operator:cdi-operator-image //cmd/cdi-controller:cdi-controller-image //cmd/cdi-apiserver:cdi-apiserver-image //cmd/cdi-cloner:cdi-cloner-image //cmd/cdi-importer:cdi-importer-image //cmd/cdi-uploadproxy:cdi-uploadproxy-image //cmd/cdi-uploadserver:cdi-uploadserver-image
 done
+
+rm -rf ${DIGESTS_DIR}/${ARCHITECTURE}
+mkdir -p ${DIGESTS_DIR}/${ARCHITECTURE}
+
+for f in $(find bazel-bin/ -name '*.digest'); do
+    dir=${DIGESTS_DIR}/${ARCHITECTURE}/$(dirname $f)
+    mkdir -p ${dir}
+    cp -f ${f} ${dir}/$(basename ${f})
+done
--- a/hack/build/bazel-generate.sh
+++ b/hack/build/bazel-generate.sh
@ -6,15 +6,15 @@ source hack/build/config.sh

 # generate BUILD files
 bazel run \
-    --config=${ARCHITECTURE} \
+    --config=${HOST_ARCHITECTURE} \
    //:gazelle $@

 if [[ "$@" =~ "vendor" ]]; then
    bazel run \
-        --config=${ARCHITECTURE} \
+        --config=${HOST_ARCHITECTURE} \
        -- @com_github_bazelbuild_buildtools//buildozer 'add clinkopts -lnbd' //$HOME/go/src/kubevirt.io/containerized-data-importer/vendor/libguestfs.org/libnbd/:go_default_library

    bazel run \
-        --config=${ARCHITECTURE} \
+        --config=${HOST_ARCHITECTURE} \
        -- @com_github_bazelbuild_buildtools//buildozer 'add copts -D_GNU_SOURCE=1' //$HOME/go/src/kubevirt.io/containerized-data-importer/vendor/libguestfs.org/libnbd/:go_default_library
 fi
--- a/hack/build/bazel-push-images.sh
+++ b/hack/build/bazel-push-images.sh
@ -47,3 +47,12 @@ bazel run \
    --define container_tag=${DOCKER_TAG} \
    --host_force_python=PY3 \
    //:push-test-images
+
+rm -rf ${DIGESTS_DIR}/${ARCHITECTURE}
+mkdir -p ${DIGESTS_DIR}/${ARCHITECTURE}
+
+for f in $(find bazel-bin/ -name '*.digest'); do
+    dir=${DIGESTS_DIR}/${ARCHITECTURE}/$(dirname $f)
+    mkdir -p ${dir}
+    cp -f ${f} ${dir}/$(basename ${f})
+done
--- a/hack/build/common.sh
+++ b/hack/build/common.sh
@ -28,6 +28,50 @@ determine_cri_bin() {
    fi
 }

+# We are formatting the architecture name here to ensure that
+# it is consistent with the platform name specified in ../.bazelrc
+# if the second argument is set, the function formats arch name for
+# image tag.
+function format_archname() {
+    local local_platform=$(uname -m)
+    local platform=$1
+    local tag=$2
+
+    if [ $# -lt 1 ]; then
+        echo ${local_platform}
+    else
+        case ${platform} in
+        x86_64 | amd64)
+            [[ $tag ]] && echo "amd64" && return
+            arch="x86_64"
+            echo ${arch}
+            ;;
+        crossbuild-aarch64 | aarch64 | arm64)
+            [[ $tag ]] && echo "arm64" && return
+            if [ ${local_platform} != "aarch64" ]; then
+                arch="crossbuild-aarch64"
+            else
+                arch="aarch64"
+            fi
+            echo ${arch}
+            ;;
+        crossbuild-s390x | s390x)
+            [[ $tag ]] && echo "s390x" && return
+            if [ ${local_platform} != "s390x" ]; then
+                arch="crossbuild-s390x"
+            else
+                arch="s390x"
+            fi
+            echo ${arch}
+            ;;
+        *)
+            echo "ERROR: invalid Arch, ${platform}, only support x86_64, aarch64 and s390x"
+            exit 1
+            ;;
+        esac
+    fi
+}
+
 CDI_DIR="$(cd $(dirname $0)/../../ && pwd -P)"
 CDI_GO_PACKAGE=kubevirt.io/containerized-data-importer
 BIN_DIR=${CDI_DIR}/bin
@ -41,7 +85,8 @@ SOURCE_DIRS="pkg tests tools"
 APIDOCS_OUT_DIR=${OUT_DIR}/apidocs
 CACHE_DIR=${OUT_DIR}/gocache
 VENDOR_DIR=${CDI_DIR}/vendor
-ARCHITECTURE="${BUILD_ARCH:-$(uname -m)}"
+DIGESTS_DIR=${OUT_DIR}/digests
+ARCHITECTURE="${ARCHITECTURE:-$(uname -m)}"
 HOST_ARCHITECTURE="$(uname -m)"
 CDI_CRI="$(determine_cri_bin)"
 if [ "${CDI_CRI}" = "docker" ]; then
--- a/hack/build/multi-arch.sh
+++ b/hack/build/multi-arch.sh
@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+#
+# This file is part of the KubeVirt project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Copyright 2023 NVIDIA CORPORATION
+#
+
+source hack/build/common.sh
+
+COMMAND=$1
+
+build_count=$(echo ${BUILD_ARCH//,/ } | wc -w)
+
+# Only add the tailing $arch when doing a multi-arch build
+if [ "$build_count" -gt 1 ]; then
+    for arch in ${BUILD_ARCH//,/ }; do
+        echo "[INFO] -- working on $arch --"
+        arch=$(format_archname $arch)
+        tag=$(format_archname $arch tag)
+        DOCKER_TAG=$DOCKER_TAG-$tag ARCHITECTURE=$arch hack/build/bazel-${COMMAND}.sh
+    done
+else
+    arch=$(format_archname ${BUILD_ARCH})
+    ARCHITECTURE=${arch} hack/build/bazel-${COMMAND}.sh
+fi
--- a/hack/build/push-container-manifest.sh
+++ b/hack/build/push-container-manifest.sh
@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+#
+# This file is part of the KubeVirt project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Copyright 2023 NVIDIA CORPORATION
+#
+
+source hack/build/common.sh
+
+# No need to push manifests if using a single arch
+build_count=$(echo ${BUILD_ARCH//,/ } | wc -w)
+if [ "$build_count" -lt 2 ]; then
+    exit 0
+fi
+
+function podman_push_manifest() {
+    image=$1
+    # FIXME: Workaround https://github.com/containers/podman/issues/18360 and remove once https://github.com/containers/podman/commit/bab4217cd16be609ac35ccf3061d1e34f787856f is released
+    echo ${CDI_CRI} manifest create ${DOCKER_PREFIX}/${image}:${DOCKER_TAG}
+    ${CDI_CRI} manifest create ${DOCKER_PREFIX}/${image}:${DOCKER_TAG}
+    for ARCH in ${BUILD_ARCH//,/ }; do
+        FORMATED_ARCH=$(format_archname ${ARCH})
+        if [ ! -f ${DIGESTS_DIR}/${FORMATED_ARCH}/bazel-bin/push-$image.digest ]; then
+            continue
+        fi
+        digest=$(cat ${DIGESTS_DIR}/${FORMATED_ARCH}/bazel-bin/push-$image.digest)
+        ${CDI_CRI} manifest add ${DOCKER_PREFIX}/${image}:${DOCKER_TAG} ${DOCKER_PREFIX}/${image}@${digest}
+    done
+    ${CDI_CRI} manifest push --all ${DOCKER_PREFIX}/${image}:${DOCKER_TAG} ${DOCKER_PREFIX}/${image}:${DOCKER_TAG}
+}
+
+function docker_push_manifest() {
+    image=$1
+    MANIFEST_IMAGES=""
+    for ARCH in ${BUILD_ARCH//,/ }; do
+        FORMATED_ARCH=$(format_archname ${ARCH})
+        if [ ! -f ${DIGESTS_DIR}/${FORMATED_ARCH}/bazel-bin/push-$image.digest ]; then
+            continue
+        fi
+        digest=$(cat ${DIGESTS_DIR}/${FORMATED_ARCH}/bazel-bin/push-$image.digest)
+        MANIFEST_IMAGES="${MANIFEST_IMAGES} --amend ${DOCKER_PREFIX}/${image}@${digest}"
+    done
+    echo ${CDI_CRI} manifest create ${DOCKER_PREFIX}/${image}:${DOCKER_TAG} ${MANIFEST_IMAGES}
+    ${CDI_CRI} manifest create ${DOCKER_PREFIX}/${image}:${DOCKER_TAG} ${MANIFEST_IMAGES}
+    ${CDI_CRI} manifest push ${DOCKER_PREFIX}/${image}:${DOCKER_TAG}
+}
+
+export DOCKER_CLI_EXPERIMENTAL=enabled
+for image in $(find ${DIGESTS_DIR}/*/bazel-bin/ -name '*.digest' -printf '%f\n' | sed s/^push-//g | sed s/\.digest$//g | sort -u); do
+    if [ "${CDI_CRI}" = "podman" ]; then
+        podman_push_manifest $image
+    else
+        docker_push_manifest $image
+    fi
+done