[6.0] d5c2696cc backport cci conf
Guillaume Quintard
guillaume at varnish-software.com
Fri Oct 23 21:57:07 UTC 2020
commit d5c2696cc836be5634d8387163d15619fb376c5e
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date: Thu Oct 15 16:27:51 2020 -0700
backport cci conf
diff --git a/.circleci/README.rst b/.circleci/README.rst
new file mode 100644
index 000000000..0c52c4235
--- /dev/null
+++ b/.circleci/README.rst
@@ -0,0 +1,93 @@
+Multiarch building, testing & packaging
+=======================================
+
+Varnish Cache uses CircleCI_ for building, testing and creating packages for
+several Linux distributions for both x86_64 and aarch64 architectures.
+
+Since CircleCI provides only x86_64 VMs the setup uses Docker and QEMU to be
+able to build, test and create packages for aarch64. This is accomplished by
+registering ``qemu-user-static`` for the CircleCI ``machine`` executor::
+
+ sudo docker run --rm --privileged multiarch/qemu-user-static --reset --credential yes --persistent yes
+
+Note 1: **--credential yes** is needed so that *setuid* flag is working.
+Without it ``sudo`` does not work in the Docker containers with architecture
+different than x86_64.
+
+Note 2: **--persistent yes** is needed so that there is no need to use
+``:register`` tag. This way one can run locally pure foreign arch Docker
+images, like the official ``arm64v8/***`` ones.
+
+With QEMU registered each build step can start a Docker image for any of the
+supported architectures to execute the ``configure``, ``make``, package steps.
+
+Workflows
+---------
+
+There are two CircleCI workflows:
+
+commit
+~~~~~~
+
+It is executed after each push to any branch, including Pull Requests
+
+The ``commit`` workflow runs two jobs:
+
+- ``dist`` - this job creates the source code distribution of Varnish Cache as
+ compressed archive (``varnish-${VERSION}.tar.gz``).
+
+- ``distcheck`` - untars the source code distribution from ``dist`` job and
+ builds (*configure*, *make*) on different Linux distributions
+
+nightly
+~~~~~~~
+
+It is executed once per day at 04:00 AM UTC time.
+
+This workflow also builds binary packages for different Linux distributions
+and CPU architectures (x86_64 & aarch64) and for this reason its run takes
+longer.
+
+It runs the following jobs:
+
+- The first two jobs that run in parallel are:
+
+ - ``tar_pkg_tools`` - this step checks out pkg-varnish-cache_ with the
+ packaging descriptions for Debian, RedHat and Alpine, and stores them in
+ the build workspace for the next steps in the pipeline.
+
+ - ``dist`` - this step creates the source code distribution of Varnish Cache
+ as compressed archive (``varnish-${VERSION}.tar.gz``). This archive is
+ also stored in the build workspace and used later by the packaging steps.
+
+
+- The next job in the workflow is ``package`` - a job that creates the
+ packages (e.g. .rpm, .deb) for each supported CPU architecture, Linux
+ distribution and its major version (e.g. *x64_centos_7*,
+ *aarch64_ubuntu_bionic*, *x64_alpine_3*, etc.). This step creates a
+ Dockerfile on the fly by using a base Docker image. This custom Docker image
+ executes a Shell script that has the recipe for creating the package for the
+ specific Linux flavor, e.g. *make-rpm-packages.sh*. The step stores the
+ packages in the build workspace.
+
+- Finally, if the previous jobs are successful, a final step is executed -
+ ``collect_packages``. This step creates an archive with all packages and
+ stores it as an artifact that can be uploaded to PackageCloud_.
+
+
+More
+----
+
+This setup can be easily extended for any CPU architectures supported by QEMU
+and for any Linux distributions which have Docker image. To do this one needs
+to add a new ``package`` job with the proper parameters for it.
+
+At the moment the setup uses *raw* Docker images and installs the required
+Linux distribution dependencies before running the tests/build/packaging code.
+This could be optimized to save some execution time by creating custom Docker
+images that extend the current ones and pre-installs the required
+dependencies.
+
+.. _CircleCI: https://app.circleci.com/pipelines/github/varnishcache/varnish-cache
+.. _pkg-varnish-cache: https://github.com/varnishcache/pkg-varnish-cache
+.. _PackageCloud: https://packagecloud.io/varnishcache/
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 7e9905b09..2efe13d56 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1,13 +1,363 @@
version: 2.1
+aliases:
+ pkg_req: &pkg_req
+ requires:
+ - dist
+ - tar_pkg_tools
+
+parameters:
+ vc-commit:
+ type: string
+ default: "HEAD"
+ pkg-commit:
+ type: string
+ default: ""
jobs:
- dummy:
+ dist:
+ description: Builds varnish-x.y.z.tar.gz that is used later for the packaging jobs
+ docker:
+ - image: centos:7
+ steps:
+ - run:
+ name: Install deps
+ command: |
+ yum install -y epel-release
+ yum install -y \
+ automake \
+ jemalloc-devel \
+ git \
+ libedit-devel \
+ libtool \
+ libunwind-devel \
+ make \
+ pcre-devel \
+ python \
+ python-sphinx
+ - checkout
+ - run:
+ name: Create the dist tarball
+ command: |
+ git checkout << pipeline.parameters.vc-commit >>
+ # if version is "trunk", it's a weekly tarball, override the version
+ if grep 'AC_INIT.*trunk.*' ./configure.ac; then
+ sed -i -e "s/^AC_INIT.*trunk.*/AC_INIT([Varnish], [$(date +%Y%m%d)], [varnish-dev at varnish-cache.org])/" ./configure.ac
+ touch .is_weekly
+ fi
+ ./autogen.des --quiet
+ make dist -j 16
+ - persist_to_workspace:
+ root: .
+ paths:
+ - .is_weekly
+ - varnish*.tar.gz
+ - tools/*.suppr
+ - .circleci
+ tar_pkg_tools:
+ description: Builds archives with the packaging tools from https://github.com/varnishcache/pkg-varnish-cache
docker:
- - image: alpine:3
+ - image: centos:7
+ steps:
+ - add_ssh_keys:
+ fingerprints:
+ - "11:ed:57:75:32:81:9d:d0:a4:5e:af:15:4b:d8:74:27"
+ - run:
+ name: Grab the pkg repo
+ command: |
+ yum install -y git
+ mkdir -p ~/.ssh
+ ssh-keyscan -H github.com >> ~/.ssh/known_hosts
+ echo ${CIRCLE_REPOSITORY_URL}
+ git clone https://github.com/varnishcache/pkg-varnish-cache.git .
+ git checkout << pipeline.parameters.pkg-commit >>
+ tar cvzf debian.tar.gz debian --dereference
+ tar cvzf redhat.tar.gz redhat --dereference
+ tar cvzf alpine.tar.gz alpine --dereference
+ - persist_to_workspace:
+ root: .
+ paths:
+ - debian.tar.gz
+ - redhat.tar.gz
+ - alpine.tar.gz
+ package:
+ parameters:
+ dist:
+ description: the Linux distribution (debian|ubuntu|centos)
+ type: string
+ release:
+ description: the release name (stretch|buster|xenial|bionic|7|8)
+ type: string
+ ext:
+ description: the package extension (deb|rpm|apk)
+ type: string
+ arch:
+ description: the architecture (x64|aarch64)
+ type: string
+ image:
+ description: the base Docker image for Dockerfile
+ type: string
+ machine:
+ image: ubuntu-1604:201903-01
steps:
- - run: echo ok
+ - attach_workspace:
+ at: ~/project
+ - run: ls -la ~/project
+ - run:
+ name: Activate QEMU
+ command: |
+ sudo docker run -it --rm --privileged multiarch/qemu-user-static --reset --credential yes --persistent yes
+ - run:
+ name: Create Dockerfile
+ command: |
+ echo "FROM << parameters.image >>" > Dockerfile
+ echo "ADD make-<< parameters.ext >>-packages.sh /usr/bin/" >> Dockerfile
+ echo 'CMD ["make-<< parameters.ext >>-packages.sh"]' >> Dockerfile
+ - run:
+ name: Build << parameters.dist >> << parameters.release >> << parameters.arch >> << parameters.ext >>
+ command: |
+ mkdir -p packages
+ cp .circleci/make-<< parameters.ext >>-packages.sh .
+ docker build -t varnish-<< parameters.ext >>-package-build:<< parameters.arch >> .
+ docker run --rm -it -e PARAM_DIST=<< parameters.dist >> -e PARAM_RELEASE=<< parameters.release >> -v$(pwd):/varnish-cache varnish-<< parameters.ext >>-package-build:<< parameters.arch >>
+ - run:
+ name: List created packages
+ command: find ./packages -name "*.<< parameters.ext >>"
+ - persist_to_workspace:
+ root: .
+ paths:
+ - "packages"
+ distcheck:
+ parameters:
+ dist:
+ description: the Linux distribution (debian|ubuntu)
+ type: string
+ release:
+ description: the release name (stretch|buster|xenial|bionic)
+ type: string
+ extra_conf:
+ description: platform-specific configure arguments
+ default: ""
+ type: string
+ docker:
+ - image: << parameters.dist >>:<< parameters.release >>
+ working_directory: /workspace
+ steps:
+ - run:
+ name: Possible << parameters.dist >>:<< parameters.release >> extra repos
+ command: |
+ if [ << parameters.dist >> = centos ]; then
+ if [ << parameters.release >> = 8 ]; then
+ dnf install -y 'dnf-command(config-manager)'
+ yum config-manager --set-enabled PowerTools
+ yum install -y diffutils python3-sphinx
+ else
+ yum install -y python-sphinx
+ fi
+ yum install -y epel-release
+ yum install -y \
+ automake \
+ jemalloc-devel \
+ libedit-devel \
+ libtool \
+ libunwind-devel \
+ make \
+ pcre-devel \
+ python3 \
+ sudo
+ elif [ << parameters.dist >> = debian -o << parameters.dist >> = ubuntu ]; then
+ export DEBIAN_FRONTEND=noninteractive
+ export DEBCONF_NONINTERACTIVE_SEEN=true
+ apt-get update
+ apt-get install -y \
+ autoconf \
+ automake \
+ build-essential \
+ ca-certificates \
+ cpio \
+ graphviz \
+ libedit-dev \
+ libjemalloc-dev \
+ libncurses-dev \
+ libpcre3-dev \
+ libtool \
+ libunwind-dev \
+ pkg-config \
+ python3-sphinx \
+ sudo
+ elif [ << parameters.dist >> = alpine ]; then
+ apk update
+ apk add -q \
+ autoconf \
+ automake \
+ build-base \
+ ca-certificates \
+ cpio \
+ gzip \
+ libedit-dev \
+ libtool \
+ libunwind-dev \
+ linux-headers \
+ pcre-dev \
+ py-docutils \
+ py3-sphinx \
+ tar \
+ sudo
+ fi
+ - attach_workspace:
+ at: /workspace
+ - run:
+ name: Extract and distcheck
+ command: |
+ tar xavf varnish-*.tar.gz --strip 1
+ if [ << parameters.dist >> = centos ]; then
+ adduser varnish
+ else
+ adduser --disabled-password --gecos "" varnish
+ fi
+
+ chown -R varnish:varnish .
+
+ export ASAN_OPTIONS=abort_on_error=1,detect_odr_violation=1,detect_leaks=1,detect_stack_use_after_return=1,detect_invalid_pointer_pairs=1,handle_segv=0,handle_sigbus=0,use_sigaltstack=0,disable_coredump=0
+ export LSAN_OPTIONS=abort_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/lsan.suppr
+ export TSAN_OPTIONS=abort_on_error=1,halt_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/tsan.suppr
+ export UBSAN_OPTIONS=halt_on_error=1,print_stacktrace=1,use_sigaltstack=0,suppressions=$(pwd)/tools/ubsan.suppr
+
+ sudo -u varnish ./configure \
+ --quiet \
+ --with-unwind \
+ --enable-developer-warnings \
+ --enable-debugging-symbols \
+ --disable-stack-protector \
+ --with-persistent-storage \
+ << parameters.extra_conf >>
+ sudo -u varnish \
+ --preserve-env=ASAN_OPTIONS,LSAN_OPTIONS,TSAN_OPTIONS,UBSAN_OPTIONS \
+ make distcheck VERBOSE=1 -j 12 -k
+
+ collect_packages:
+ docker:
+ - image: centos:7
+ steps:
+ - attach_workspace:
+ at: ~/project
+ - run: ls -la ~/project/
+ - run:
+ name: Tar the packages
+ command: |
+ tar cvzf packages.tar.gz packages
+ - store_artifacts:
+ destination: packages.tar.gz
+ path: packages.tar.gz
+
workflows:
- build:
+ version: 2
+ commit:
jobs:
- - dummy
+ - distcheck:
+ name: distcheck_centos_7
+ dist: centos
+ release: "7"
+ requires:
+ - dist
+ - distcheck:
+ name: distcheck_debian_buster
+ dist: debian
+ release: buster
+ extra_conf: --enable-asan --enable-ubsan
+ requires:
+ - dist
+ - dist
+ - tar_pkg_tools
+ - package:
+ name: aarch64-ubuntu-bionic
+ dist: ubuntu
+ release: bionic
+ arch: aarch64
+ image: arm64v8/ubuntu:bionic
+ ext: deb
+ <<: *pkg_req
+ - package:
+ name: x64-ubuntu-bionic
+ dist: ubuntu
+ release: bionic
+ arch: x64
+ image: ubuntu:bionic
+ ext: deb
+ <<: *pkg_req
+ - package:
+ name: aarch64-ubuntu-xenial
+ dist: ubuntu
+ release: xenial
+ arch: aarch64
+ image: arm64v8/ubuntu:xenial
+ ext: deb
+ <<: *pkg_req
+ - package:
+ name: x64-ubuntu-xenial
+ dist: ubuntu
+ release: xenial
+ arch: x64
+ image: ubuntu:xenial
+ ext: deb
+ <<: *pkg_req
+ - package:
+ name: aarch64-debian-buster
+ dist: debian
+ release: buster
+ arch: aarch64
+ image: arm64v8/debian:buster-slim
+ ext: deb
+ <<: *pkg_req
+ - package:
+ name: x64-debian-buster
+ dist: debian
+ release: buster
+ arch: x64
+ image: debian:buster-slim
+ ext: deb
+ <<: *pkg_req
+ - package:
+ name: aarch64-debian-stretch
+ dist: debian
+ release: stretch
+ arch: aarch64
+ image: arm64v8/debian:stretch-slim
+ ext: deb
+ <<: *pkg_req
+ - package:
+ name: x64-debian-stretch
+ dist: debian
+ release: stretch
+ arch: x64
+ image: debian:stretch-slim
+ ext: deb
+ <<: *pkg_req
+ - package:
+ name: aarch64-centos-7
+ dist: centos
+ release: "7"
+ arch: aarch64
+ image: arm64v8/centos:7
+ ext: rpm
+ <<: *pkg_req
+ - package:
+ name: x64-centos-7
+ dist: centos
+ release: "7"
+ arch: x64
+ image: centos:7
+ ext: rpm
+ <<: *pkg_req
+ - collect_packages:
+ requires:
+ - x64-ubuntu-xenial
+ - aarch64-ubuntu-xenial
+ - x64-ubuntu-bionic
+ - aarch64-ubuntu-bionic
+ - x64-debian-stretch
+ - aarch64-debian-stretch
+ - x64-debian-buster
+ - aarch64-debian-buster
+ - x64-centos-7
+ - aarch64-centos-7
diff --git a/.circleci/make-apk-packages.sh b/.circleci/make-apk-packages.sh
new file mode 100755
index 000000000..26b672c2c
--- /dev/null
+++ b/.circleci/make-apk-packages.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env sh
+
+set -eux
+
+apk add -q --no-progress --update tar alpine-sdk
+
+echo "PARAM_RELEASE: $PARAM_RELEASE"
+echo "PARAM_DIST: $PARAM_DIST"
+
+if [ -z "$PARAM_RELEASE" ]; then
+ echo "Env variable PARAM_RELEASE is not set! For example PARAM_RELEASE=8, for CentOS 8"
+ exit 1
+elif [ -z "$PARAM_DIST" ]; then
+ echo "Env variable PARAM_DIST is not set! For example PARAM_DIST=centos"
+ exit 1
+fi
+
+cd /varnish-cache
+tar xazf alpine.tar.gz --strip 1
+
+adduser -D builder
+echo "builder ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers
+addgroup builder abuild
+mkdir -p /var/cache/distfiles
+chmod -R a+w /var/cache/distfiles
+
+echo "Generate key"
+su builder -c "abuild-keygen -nai"
+
+echo "Fix APKBUILD's variables"
+tar xavf varnish-*.tar.gz
+VERSION=$(varnish-*/configure --version | awk 'NR == 1 {print $NF}')
+echo "Version: $VERSION"
+sed -i "s/@VERSION@/$VERSION/" APKBUILD
+rm -rf varnish-*/
+
+echo "Change the ownership so that abuild is able to write its logs"
+chown builder -R .
+echo "Fix checksums, build"
+su builder -c "abuild checksum"
+su builder -c "abuild -r"
+
+echo "Fix the APKBUILD's version"
+su builder -c "mkdir apks"
+ARCH=`uname -m`
+su builder -c "cp /home/builder/packages/$ARCH/*.apk apks"
+
+echo "Import the packages into the workspace"
+mkdir -p packages/$PARAM_DIST/$PARAM_RELEASE/$ARCH/
+mv /home/builder/packages/$ARCH/*.apk packages/$PARAM_DIST/$PARAM_RELEASE/$ARCH/
+
+echo "Allow to read the packages by 'circleci' user outside of Docker after 'chown builder -R .' above"
+chmod -R a+rwx .
diff --git a/.circleci/make-deb-packages.sh b/.circleci/make-deb-packages.sh
new file mode 100755
index 000000000..3b3ba0847
--- /dev/null
+++ b/.circleci/make-deb-packages.sh
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+set -eux
+
+export DEBIAN_FRONTEND=noninteractive
+export DEBCONF_NONINTERACTIVE_SEEN=true
+apt-get update
+apt-get install -y dpkg-dev debhelper devscripts equivs pkg-config apt-utils fakeroot
+
+echo "PARAM_RELEASE: $PARAM_RELEASE"
+echo "PARAM_DIST: $PARAM_DIST"
+
+
+if [ -z "$PARAM_RELEASE" ]; then
+ echo "Env variable PARAM_RELEASE is not set! For example PARAM_RELEASE=8, for CentOS 8"
+ exit 1
+elif [ -z "$PARAM_DIST" ]; then
+ echo "Env variable PARAM_DIST is not set! For example PARAM_DIST=centos"
+ exit 1
+fi
+
+# Ubuntu 20.04 aarch64 fails when using fakeroot-sysv with:
+# semop(1): encountered an error: Function not implemented
+update-alternatives --set fakeroot /usr/bin/fakeroot-tcp
+
+cd /varnish-cache
+ls -la
+
+echo "Untar debian..."
+tar xavf debian.tar.gz
+
+echo "Untar orig..."
+tar xavf varnish-*.tar.gz --strip 1
+
+echo "Update changelog version..."
+if [ -e .is_weekly ]; then
+ WEEKLY='-weekly'
+else
+ WEEKLY=
+fi
+VERSION=$(./configure --version | awk 'NR == 1 {print $NF}')$WEEKLY~$PARAM_RELEASE
+sed -i -e "s|@VERSION@|$VERSION-1|" "debian/changelog"
+
+echo "Install Build-Depends packages..."
+yes | mk-build-deps --install debian/control || true
+
+echo "Build the packages..."
+dpkg-buildpackage -us -uc -j16
+
+echo "Prepare the packages for storage..."
+mkdir -p packages/$PARAM_DIST/$PARAM_RELEASE/
+mv ../*.deb packages/$PARAM_DIST/$PARAM_RELEASE/
+
+if [ "`uname -m`" = "x86_64" ]; then
+ ARCH="amd64"
+else
+ ARCH="arm64"
+fi
+
+DSC_FILE=$(ls ../*.dsc)
+DSC_FILE_WO_EXT=$(basename ${DSC_FILE%.*})
+mv $DSC_FILE packages/$PARAM_DIST/$PARAM_RELEASE/${DSC_FILE_WO_EXT}_${ARCH}.dsc
diff --git a/.circleci/make-rpm-packages.sh b/.circleci/make-rpm-packages.sh
new file mode 100755
index 000000000..b443c9139
--- /dev/null
+++ b/.circleci/make-rpm-packages.sh
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+set -eux
+
+echo "PARAM_RELEASE: $PARAM_RELEASE"
+echo "PARAM_DIST: $PARAM_DIST"
+
+if [ -z "$PARAM_RELEASE" ]; then
+ echo "Env variable PARAM_RELEASE is not set! For example PARAM_RELEASE=8, for CentOS 8"
+ exit 1
+elif [ -z "$PARAM_DIST" ]; then
+ echo "Env variable PARAM_DIST is not set! For example PARAM_DIST=centos"
+ exit 1
+fi
+
+yum install -y epel-release
+
+if [ "$PARAM_DIST" = centos ]; then
+ if [ "$PARAM_RELEASE" = 8 ]; then
+ dnf install -y 'dnf-command(config-manager)'
+ yum config-manager --set-enabled PowerTools
+ fi
+fi
+
+yum install -y rpm-build yum-utils
+
+export DIST_DIR=build
+
+cd /varnish-cache
+rm -rf $DIST_DIR
+mkdir $DIST_DIR
+
+
+echo "Untar redhat..."
+tar xavf redhat.tar.gz -C $DIST_DIR
+
+echo "Untar orig..."
+tar xavf varnish-*.tar.gz -C $DIST_DIR --strip 1
+
+echo "Build Packages..."
+if [ -e .is_weekly ]; then
+ WEEKLY='.weekly'
+else
+ WEEKLY=
+fi
+VERSION=$("$DIST_DIR"/configure --version | awk 'NR == 1 {print $NF}')$WEEKLY
+
+cp -r -L "$DIST_DIR"/redhat/* "$DIST_DIR"/
+tar zcf "$DIST_DIR.tgz" --exclude "$DIST_DIR/redhat" "$DIST_DIR"/
+
+RPMVERSION="$VERSION"
+
+RESULT_DIR="rpms"
+CUR_DIR="$(pwd)"
+
+rpmbuild() {
+ command rpmbuild \
+ --define "_smp_mflags -j10" \
+ --define "_sourcedir $CUR_DIR" \
+ --define "_srcrpmdir $CUR_DIR/${RESULT_DIR}" \
+ --define "_rpmdir $CUR_DIR/${RESULT_DIR}" \
+ --define "versiontag ${RPMVERSION}" \
+ --define "releasetag 0.0" \
+ --define "srcname $DIST_DIR" \
+ --define "nocheck 1" \
+ "$@"
+}
+
+yum-builddep -y "$DIST_DIR"/redhat/varnish.spec
+rpmbuild -bs "$DIST_DIR"/redhat/varnish.spec
+rpmbuild --rebuild "$RESULT_DIR"/varnish-*.src.rpm
+
+echo "Prepare the packages for storage..."
+mkdir -p packages/$PARAM_DIST/$PARAM_RELEASE/
+mv rpms/*/*.rpm packages/$PARAM_DIST/$PARAM_RELEASE/
More information about the varnish-commit
mailing list