github/netdata
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Consolidate build matrix across CI jobs. (#11807)

Browse Source 
* Restructure CI build matrix handling for better consolidation.

* Update core packaging code to use new distro.yml file.

* Update repository config packaging code to use new distro.yml file.
This commit is contained in:
Austin S. Hemmelgarn 2022-02-16 08:53:22 -05:00 committed by GitHub
parent 598ce2180a
commit 46d73e34c0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 255 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
.github/data/build-matrix.json vendored
View File

@ -1,98 +0,0 @@
{
"include": [
{
"distro": "alpine:edge",
"artifact_key": "alpine-edge",
"pre": "apk add -U bash",
"rmjsonc": "apk del json-c-dev"
},
{
"distro": "alpine:3.14",
"artifact_key": "alpine-3.14",
"pre": "apk add -U bash",
"rmjsonc": "apk del json-c-dev"
},
{
"distro": "alpine:3.13",
"artifact_key": "alpine-3.13",
"pre": "apk add -U bash",
"rmjsonc": "apk del json-c-dev"
},
{
"distro": "alpine:3.12",
"artifact_key": "alpine-3.12",
"pre": "apk add -U bash",
"rmjsonc": "apk del json-c-dev"
},
{
"distro": "archlinux:latest",
"artifact_key": "archlinux",
"pre": "pacman --noconfirm -Syu && pacman --noconfirm -Sy grep libffi"
},
{
"distro": "centos:7",
"artifact_key": "centos7"
},
{
"distro": "rockylinux/rockylinux:8",
"artifact_key": "rockylinux8",
"rmjsonc": "dnf remove -y json-c-devel"
},
{
"distro": "debian:11",
"artifact_key": "debian11",
"pre": "apt-get update",
"rmjsonc": "apt-get remove -y libjson-c-dev"
},
{
"distro": "debian:10",
"artifact_key": "debian10",
"pre": "apt-get update",
"rmjsonc": "apt-get remove -y libjson-c-dev"
},
{
"distro": "debian:9",
"artifact_key": "debian9",
"pre": "apt-get update",
"rmjsonc": "apt-get remove -y libjson-c-dev"
},
{
"distro": "fedora:35",
"artifact_key": "fedora35",
"rmjsonc": "dnf remove -y json-c-devel"
},
{
"distro": "fedora:34",
"artifact_key": "fedora34",
"rmjsonc": "dnf remove -y json-c-devel"
},
{
"distro": "opensuse/leap:15.3",
"artifact_key": "opensuse15.3",
"rmjsonc": "zypper rm -y libjson-c-devel"
},
{
"distro": "oraclelinux:8",
"artifact_key": "oraclelinux8",
"rmjsonc": "dnf remove -y json-c-devel"
},
{
"distro": "ubuntu:21.10",
"artifact_key": "ubuntu21.10",
"pre": "rm -f /etc/apt/apt.conf.d/docker && apt-get update",
"rmjsonc": "apt-get remove -y libjson-c-dev"
},
{
"distro": "ubuntu:20.04",
"artifact_key": "ubuntu20.04",
"pre": "apt-get update",
"rmjsonc": "apt-get remove -y libjson-c-dev"
},
{
"distro": "ubuntu:18.04",
"artifact_key": "ubuntu18.04",
"pre": "apt-get update",
"rmjsonc": "apt-get remove -y libjson-c-dev"
}
]
}

145
.github/data/distros.yml vendored Normal file
View File

@ -0,0 +1,145 @@
# This defines the full set of distros we run CI on.
---
platform_map: # map packaging architectures to docker platforms
i386: linux/i386
amd64: linux/amd64
armhf: linux/arm/v7
arm64: linux/arm64/v8
include:
- &alpine
distro: alpine
version: edge
env_prep: |
apk add -U bash
jsonc_removal: |
apk del json-c-dev
- <<: *alpine
version: "3.15"
- <<: *alpine
version: "3.14"
- <<: *alpine
version: "3.13"
- <<: *alpine
version: "3.12"
- distro: archlinux
version: latest
env_prep: |
pacman --noconfirm -Syu && pacman --noconfirm -Sy grep libffi
- distro: rockylinux
version: "8"
base_image: rockylinux/rockylinux
jsonc_removal: |
dnf remove -y json-c-devel
packages:
type: rpm
repo_distro: el/8
arches:
- amd64
- arm64
- distro: centos
version: "7"
packages:
type: rpm
repo_distro: el/7
arches:
- amd64
- &debian
distro: debian
version: "11"
env_prep: |
apt-get update
jsonc_removal: |
apt-get purge -y libjson-c-dev
packages: &debian_packages
type: deb
repo_distro: debian/bullseye
arches:
- i386
- amd64
- armhf
- arm64
- <<: *debian
version: "10"
packages:
<<: *debian_packages
repo_distro: debian/buster
- <<: *debian
version: "9"
packages:
<<: *debian_packages
repo_distro: debian/stretch
- &fedora
distro: fedora
version: "35"
jsonc_removal: |
dnf remove -y json-c-devel
packages: &fedora_packages
type: rpm
repo_distro: fedora/35
arches:
- amd64
- armhf
- arm64
- <<: *fedora
version: "34"
packages:
<<: *fedora_packages
repo_distro: fedora/34
- &opensuse
distro: opensuse
version: "15.3"
base_image: opensuse/leap
jsonc_removal: |
zypper rm -y libjson-c-devel
packages:
type: rpm
repo_distro: opensuse/15.3
arches:
- amd64
- arm64
- distro: oraclelinux
version: "8"
jsonc_removal: |
dnf remove -y json-c-devel
packages:
type: rpm
repo_distro: ol/8
arches:
- amd64
- arm64
- &ubuntu
distro: ubuntu
version: "21.10"
env_prep: |
rm -f /etc/apt/apt.conf.d/docker && apt-get update
jsonc_removal: |
apt-get remove -y libjson-c-dev
packages: &ubuntu_packages
type: deb
repo_distro: ubuntu/impish
arches:
- amd64
- armhf
- arm64
- <<: *ubuntu
version: "20.04"
packages:
<<: *ubuntu_packages
repo_distro: ubuntu/focal
- <<: *ubuntu
version: "18.04"
packages:
<<: *ubuntu_packages
repo_distro: ubuntu/bionic
arches:
- i386
- amd64
- armhf
- arm64

30
.github/workflows/build.yml vendored
View File

@ -129,13 +129,29 @@ jobs:
uses: actions/checkout@v2
- name: Prepare tools
run: |
sudo apt-get update && sudo apt-get install -y jq
sudo apt-get update && sudo apt-get install -y python3-ruamel.yaml
- name: Read build matrix
id: set-matrix
shell: python3 {0}
run: |
TASKS="$(jq -c . .github/data/build-matrix.json)"
echo "Generated Matrix: $TASKS"
echo "::set-output name=matrix::$TASKS"
from ruamel.yaml import YAML
import json
yaml = YAML(typ='safe')
with open('.github/data/distros.yml') as f:
data = yaml.load(f)
del data['platform_map']
for i, v in enumerate(data['include']):
data['include'][i]['artifact_key'] = data['include'][i]['distro'] + str(data['include'][i]['version']).replace('.', '')
if 'packages' in data['include'][i]:
del data['include'][i]['packages']
if 'base_image' in data['include'][i]:
data['include'][i]['distro'] = data['include'][i]['base_image']
del data['include'][i]['base_image']
data['include'][i]['distro'] = ':'.join([data['include'][i]['distro'], str(data['include'][i]['version'])])
del data['include'][i]['version']
matrix = json.dumps(data, sort_keys=True)
print('Generated Matrix: ' + matrix)
print('::set-output name=matrix::' + matrix)
prepare-test-images: # Prepare the test environments for our build checks. This also checks dependency handling code for each tested environment.
name: Prepare Test Environments
@ -160,8 +176,8 @@ jobs:
file: .github/dockerfiles/Dockerfile.build_test
build-args: |
BASE=${{ matrix.distro }}
PRE=${{ matrix.pre }}
RMJSONC=${{ matrix.rmjsonc }}
PRE=${{ matrix.env_prep }}
RMJSONC=${{ matrix.jsonc_removal }}
outputs: type=oci,dest=/tmp/image.tar
tags: test:${{ matrix.artifact_key }}
- name: Upload image artifact
@ -224,7 +240,7 @@ jobs:
docker run --security-opt seccomp=unconfined -w /netdata sha256:${{ steps.load.outputs.image }} \
/bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud'
- name: netdata-installer on ${{ matrix.distro }}, require cloud, no JSON-C
if: matrix.rmjsonc != ''
if: matrix.jsonc_removal != ''
run: |
docker run --security-opt seccomp=unconfined -w /netdata sha256:${{ steps.load.outputs.image }} \
/bin/sh -c '/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud'

81
.github/workflows/packaging.yml vendored
View File

@ -21,49 +21,56 @@ concurrency:
group: packages-${{ github.ref }}-${{ github.event_name }}
cancel-in-progress: true
jobs:
matrix:
name: Prepare Build Matrix
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Prepare tools
run: |
sudo apt-get update && sudo apt-get install -y python3-ruamel.yaml
- name: Read build matrix
id: set-matrix
shell: python3 {0}
run: |
from ruamel.yaml import YAML
import json
yaml = YAML(typ='safe')
entries = list()
with open('.github/data/distros.yml') as f:
data = yaml.load(f)
for i, v in enumerate(data['include']):
if 'packages' in data['include'][i]:
for arch in data['include'][i]['packages']['arches']:
entries.append({
'distro': data['include'][i]['distro'],
'version': data['include'][i]['version'],
'pkgclouddistro': data['include'][i]['packages']['repo_distro'],
'format': data['include'][i]['packages']['type'],
'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else data['include'][i]['distro'],
'platform': data['platform_map'][arch],
'arch': arch
})
entries.sort(key=lambda k: (k['arch'], k['distro'], k['version']))
matrix = json.dumps({'include': entries}, sort_keys=True)
print('Generated Matrix: ' + matrix)
print('::set-output name=matrix::' + matrix)
build:
name: Build
runs-on: ubuntu-latest
env:
DOCKER_CLI_EXPERIMENTAL: enabled
needs:
- matrix
strategy:
matrix:
include:
- {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
- {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/i386, arch: i386}
- {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/arm/v7, arch: armhf}
- {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/arm64/v8, arch: arm64}
- {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
- {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/i386, arch: i386}
- {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/arm/v7, arch: armhf}
- {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/arm64/v8, arch: arm64}
- {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/amd64, arch: amd64, alias: bullseye}
- {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/i386, arch: i386, alias: bullseye}
- {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/arm/v7, arch: armhf, alias: bullseye}
- {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/arm64/v8, arch: arm64, alias: bullseye}
- {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
- {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/i386, arch: i386}
- {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf}
- {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64}
- {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
- {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf}
- {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64}
- {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
- {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf}
- {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64}
- {distro: centos, version: "7", pkgclouddistro: el/7, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
- {distro: rockylinux, version: "8", pkgclouddistro: el/8, format: rpm, base_image: rockylinux/rockylinux, platform: linux/amd64, arch: amd64}
- {distro: rockylinux, version: "8", pkgclouddistro: el/8, format: rpm, base_image: rockylinux/rockylinux, platform: linux/arm64/v8, arch: arm64}
- {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
- {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/arm/v7, arch: armhf}
- {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/arm64/v8, arch: arm64}
- {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
- {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/arm/v7, arch: armhf}
- {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/arm64/v8, arch: arm64}
- {distro: opensuse, version: "15.3", pkgclouddistro: opensuse/15.3, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64}
- {distro: opensuse, version: "15.3", pkgclouddistro: opensuse/15.3, format: rpm, base_image: opensuse/leap, platform: linux/arm64/v8, arch: arm64}
- {distro: oraclelinux, version: "8", pkgclouddistro: ol/8, format: rpm, base_image: oraclelinux, platform: linux/amd64, arch: amd64}
- {distro: oraclelinux, version: "8", pkgclouddistro: ol/8, format: rpm, base_image: oraclelinux, platform: linux/arm64/v8, arch: arm64}
matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
# We intentiaonally disable the fail-fast behavior so that a
# build failure for one version doesn't prevent us from publishing
# successfully built and tested packages for another version.

58
.github/workflows/repoconfig-packages.yml vendored
View File

@ -16,28 +16,56 @@ on:
env:
DISABLE_TELEMETRY: 1
jobs:
matrix:
name: Prepare Build Matrix
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Prepare tools
run: |
sudo apt-get update && sudo apt-get install -y python3-ruamel.yaml
- name: Read build matrix
id: set-matrix
shell: python3 {0}
run: |
from ruamel.yaml import YAML
import json
yaml = YAML(typ='safe')
entries = list()
with open('.github/data/distros.yml') as f:
data = yaml.load(f)
for i, v in enumerate(data['include']):
if 'packages' in data['include'][i]:
entries.append({
'distro': data['include'][i]['distro'],
'version': data['include'][i]['version'],
'pkgclouddistro': data['include'][i]['packages']['repo_distro'],
'format': data['include'][i]['packages']['type'],
'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else data['include'][i]['distro'],
'platform': data['platform_map']['amd64'],
'arch': 'amd64'
})
entries.sort(key=lambda k: (k['arch'], k['distro'], k['version']))
matrix = json.dumps({'include': entries}, sort_keys=True)
print('Generated Matrix: ' + matrix)
print('::set-output name=matrix::' + matrix)
build:
name: Build
runs-on: ubuntu-latest
env:
DISABLE_TELEMETRY: 1
DOCKER_CLI_EXPERIMENTAL: enabled
needs:
- matrix
strategy:
# This needs to be kept in sync with the matrix in packaging.yml, but should only include the AMD64 lines.
matrix:
include:
- {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
- {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
- {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
- {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
- {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
- {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
- {distro: centos, version: "7", pkgclouddistro: el/7, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
- {distro: rockylinux, version: "8", pkgclouddistro: el/8, format: rpm, base_image: rockylinux/rockylinux, platform: linux/amd64, arch: amd64}
- {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
- {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
- {distro: opensuse, version: "15.3", pkgclouddistro: opensuse/15.3, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64}
- {distro: oraclelinux, version: "8", pkgclouddistro: ol/8, format: rpm, base_image: oraclelinux, platform: linux/amd64, arch: amd64}
matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
# We intentiaonally disable the fail-fast behavior so that a
# build failure for one version doesn't prevent us from publishing
# successfully built and tested packages for another version.