NLnetLabs-krill/.github/workflows/pkg.yml

# GitHub Actions workflow for building and testing Krill O/S packages.
# Uses GitHub Actions caching to avoid rebuilding Rust cargo-deb and
# Krill dependencies on every run.
#
# Note: at the time of writing the GH cache contents expire after a
# week if not used so the next build may be much slower as it will
# have to re-download/build/install lots of Rust crates.
#
# Packages are built inside Docker containers as GH Runners have extra libraries
# and packages installed which can cause package building to succeed but package
# installation on a real target O/S to fail, due to being built against too
# recent version of a package such as libssl or glibc.
#
# Packages are tested inside LXC/LXD containers because Docker containers don't
# by default support init managers such as systemd but we want to test systemd
# service unit installation and activation.

name: Packaging
on:
  push:
    branches:
    - master
    paths-ignore:
      - '.dockerignore'
      - '.github/workflow/pkg.yml'
      - 'Changelog.md'
      - 'Dockerfile'
      - 'doc/**'
      - 'docker/**'
      - 'LICENSE'
      - 'README.md'
      - 'tests/e2e/**'
  # Hmm, annoying, do we really have to duplicate this?
  pull_request:
    branches:
    - master
    paths-ignore:
      - '.dockerignore'
      - '.github/workflow/pkg.yml'
      - 'Changelog.md'
      - 'Dockerfile'
      - 'doc/**'
      - 'docker/**'
      - 'LICENSE'
      - 'README.md'
      - 'tests/e2e/**'

defaults:
  run:
    # see: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
    shell: bash --noprofile --norc -eo pipefail -x {0}

jobs:
  # Use the cargo-deb Rust create to build a Debian package for installing
  # Krill. See: https://github.com/mmstick/cargo-deb
  deb-pkg:
    strategy:
      matrix:
        image: [
          "ubuntu:16.04",
          "ubuntu:18.04",
          "ubuntu:20.04",
          "debian:9",
          "debian:10",
        ]
    env:
      CARGO_DEB_VER: 05545d4
      # A Krill version of the form 'x.y.z-plus' denotes a dev build that is
      # newer than the released x.y.z version but is not yet a new release.
      NEXT_VER_LABEL: plus
    name: deb-pkg
    runs-on: ubuntu-latest
    # Build on the oldest platform we are targeting in order to avoid
    # https://github.com/rust-lang/rust/issues/57497. Specifying container
    # causes all of the steps in this job to run inside a Docker container.
    container: ${{ matrix.image }}
    steps:
    # Set an environment variable that will be available to later steps in
    # run commands, and a GH Actions output variable that can be used in later
    # step definitions.
    - name: Set vars
      id: setvars
      shell: bash
      run: |
        echo ::set-env name=DEB_NAME::$(echo $MATRIX_IMAGE | tr -d ':.')
        echo ::set-output name=pkgname::$(echo $MATRIX_IMAGE | tr -d ':.')
      env:
        MATRIX_IMAGE: ${{ matrix.image }}

    # Git clone the Krill code in the branch we were invoked on.
    - name: Checkout repository
      uses: actions/checkout@v1

    # Install Rust the hard way rather than using a GH Action because the action
    # doesn't work inside a Docker container.
    - name: Install Rust
      run: |
        apt-get update
        apt-get install -y curl
        curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --profile minimal -y
        echo "::add-path::$HOME/.cargo/bin"
      env:
        DEBIAN_FRONTEND: noninteractive

    - name: Install compilation and other dependencies
      run: |
        apt-get install -y build-essential jq libssl-dev lintian pkg-config
      env:
        DEBIAN_FRONTEND: noninteractive

    # Speed up Krill Rust builds by caching unchanged built dependencies.
    # See: https://github.com/actions/cache/blob/master/examples.md#rust---cargo
    - name: Cache Dot Cargo
      uses: actions/cache@v2
      with:
        path: |
          ~/.cargo/registry
          ~/.cargo/git
          target
        key: ${{ job.container.image }}-${{ matrix.image }}-cargo-${{ hashFiles('**/Cargo.lock') }}

    # Speed up cargo-deb installation by only re-downloading and re-building its
    # dependent crates if we change the version of cargo-deb that we are using.
    - name: Cache Cargo Deb binary
      id: cache-cargo-deb
      uses: actions/cache@v2
      with:
        path: ~/.cargo/bin/cargo-deb
        key: ${{ job.container.image }}-${{ matrix.image }}-cargo-deb-${{ env.CARGO_DEB_VER }}

    # Only install cargo-deb if not already fetched from the cache.
    - name: Install Cargo Deb
      if: steps.cache-cargo-deb.outputs.cache-hit != 'true'
      run: |
        cargo install --git https://github.com/ximon18/cargo-deb.git --branch dh_installsystemd_and_changelog_fix --rev $CARGO_DEB_VER cargo-deb

    # Instruct cargo-deb to build the Debian package using the config section
    # in Cargo.toml for the specified "variant".
    - name: Create the DEB package
      run: |
        # Packages for different distributions (e.g. Stretch, Buster) of the same
        # O/S (e.g. Debian) when served from a single package repository MUST have
        # unique package_ver_architecture triples. Cargo deb can vary the name based
        # on the 'variant' config section in use, but doesn't do so according to
        # Debian policy (as it modifies the package name, not the package version).
        #   Format: package_ver_architecture
        #   Where ver has format: [epoch:]upstream_version[-debian_revision]
        #   And debian_version should be of the form: 1<xxx>
        #   Where it is common to set <xxx> to the O/S name.
        # See:
        #   - https://unix.stackexchange.com/a/190899
        #   - https://www.debian.org/doc/debian-policy/ch-controlfields.html#version
        #   - https://readme.phys.ethz.ch/documentation/debian_version_numbers/
        # Therefore we generate the version ourselves.
        #
        # In addition, Semantic Versioning and Debian version policy cannot
        # express a pre-release label in the same way. For example 0.8.0-rc.1
        # is a valid Cargo.toml [package].version value but when used as a
        # Debian package version 0.8.0-rc.1 would be considered _NEWER_ than
        # the final 0.8.0 release. To express this in a Debian compatible way we
        # must replace the dash '-' with a tilda '~'.
        #
        # Finally, sometimes we want a version to be NEWER than the latest
        # release but without having to decide what higher semver number to bump
        # to. In this case we do NOT want dash '-' to become '~' because `-`
        # is treated as higher and tilda is treated as lower.
        KRILL_VER=$(cargo read-manifest | jq -r '.version' | tr '-' '~')
        DEB_KRILL_VER=$(echo $KRILL_VER | sed -e "s/~$NEXT_VER_LABEL/-$NEXT_VER_LABEL/")
        case ${MATRIX_IMAGE} in
          ubuntu:16.04) OS_REL=xenial ;;
          ubuntu:18.04) OS_REL=bionic ;;
          ubuntu:20.04) OS_REL=focal ;;
          debian:9)     OS_REL=stretch ;;
          debian:10)    OS_REL=buster ;;
          *)            echo 2>&1 "ERROR: Unexpected matrix image"; exit 1 ;;
        esac

        case ${{ github.event_name }} in
          pull_request) MAINTAINER="${{ github.actor }} <unknown@email.address>" ;;
          push)         MAINTAINER="${{ github.event.pusher.name }} <${{ github.event.pusher.email }}>" ;;
          *)            echo 2>&1 "ERROR: Unexpected GitHub Actions event"; exit 1 ;;
        esac

        # Generate the changelog file that Debian packages are required to have.
        # See: https://www.debian.org/doc/manuals/maint-guide/dreq.en.html#changelog
        echo "krill (${DEB_KRILL_VER}) unstable; urgency=medium" >debian/changelog
        echo "  * See: https://github.com/NLnetLabs/krill/releases/tag/v${KRILL_VER}" >>debian/changelog
        echo " -- maintainer ${MAINTAINER}  $(date --rfc-email)" >>debian/changelog

        DEB_VER="${DEB_KRILL_VER}-1${OS_REL}"
        cargo deb --variant $DEB_NAME --deb-version $DEB_VER -v
      env:
        MATRIX_IMAGE: ${{ matrix.image }}

    # See what Lintian thinks of our package but don't fail the build because of
    # it. E.g. Cargo Deb doesn't compress man pages but Lintian (and Debian
    # Policy) say they should be compressed.
    - name: Verify the DEB package
      run: |
        lintian -v target/debian/*.deb

    # Upload the produced DEB package. The artifact will be available
    # via the GH Actions job summary and build log pages, but only to
    # users logged in to GH with sufficient rights in this project. The
    # uploaded artifact is also downloaded by the next job (see below)
    # to sanity check that it can be installed and results in a working
    # Krill installation.
    - name: Upload DEB package
      uses: actions/upload-artifact@v2
      with:
        name: ${{ steps.setvars.outputs.pkgname }}
        path: target/debian/*.deb

  # Download and sanity check on target operating systems the packages created
  # by previous jobs (see above). Don't test on GH runners as they come with
  # lots of software and libraries pre-installed and thus are not representative
  # of the actual deployment targets, nor do GH runners support all targets that
  # we want to test. Don't test in Docker containers as they do not support
  # systemd.
  deb-pkg-test:
    name: deb-pkg-test
    needs: deb-pkg
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        image:
          - 'ubuntu:16.04'
          - 'ubuntu:18.04'
          - 'ubuntu:20.04'
          - 'debian:9'
          - 'debian:10'
        mode:
          - 'fresh-install'
          - 'upgrade-from-published'
    steps:
    # Set some environment variables that will be available to "run" steps below
    # in this job, and some output variables that will be available in GH Action
    # step definitions below.
    - name: Set vars
      id: setvars
      shell: bash
      run: |
        if [[ $MATRIX_IMAGE == *debian* ]]; then
          SLASHED=$(echo $MATRIX_IMAGE | tr ':' '/')
          echo ::set-env name=LXC_IMAGE::$(echo "images:${SLASHED}/cloud")
        else
          echo ::set-env name=LXC_IMAGE::$(echo $MATRIX_IMAGE)
        fi
        echo ::set-env name=DEB_NAME::$(echo $MATRIX_IMAGE | tr -d ':.')
        echo ::set-output name=pkgname::$(echo $MATRIX_IMAGE | tr -d ':.')
      env:
        MATRIX_IMAGE: ${{ matrix.image }}

    - name: Download DEB package
      uses: actions/download-artifact@v2
      with:
        name: ${{ steps.setvars.outputs.pkgname }}

    - name: Add current user to LXD group
      run: |
        sudo usermod --append --groups lxd $(whoami)

    - name: Initialize LXD
      run: |
        sudo lxd init --auto

    - name: Check LXD configuration
      run: |
        sg lxd -c "lxc info"

    - name: Launch LXC container
      run: |

        # security.nesting=true is needed to avoid error "Failed to set up mount
        # namespacing: Permission denied" in a Debian 10 container.
        sg lxd -c "lxc launch $LXC_IMAGE -c security.nesting=true testcon"

    # Run apt-get update and install packages missing in some O/S images that
    # are needed by later steps, but first wait for cloud-init to finish
    # otherwise the network isn't yet ready.
    - name: Prepare container
      shell: bash
      run: |
        while true; do
          case ${LXC_IMAGE} in
            # ubuntu:16.04|ubuntu:18.04|ubuntu:20.04|images:debian/10/cloud)
            ubuntu:16.04|ubuntu:18.04|ubuntu:20.04|images)
            OUTPUT=$(sg lxd -c "lxc exec testcon -- cloud-init status")
              [[ "$OUTPUT" == "status: done" ]] && break
              ;;
            # images:debian/9/cloud)
            #   [ -f /run/cloud-init/result.json ] && echo "Debian 9 extra pause" && sleep 2s && break
            
            images:debian/9/cloud|images:debian/10/cloud)
              # Not sure why the above don't work for Debian 9 and 10. Just
              # sleep for now instead to avoid the name resolution failures that
              # otherwise happen during apt-get update below.
              sleep 60s && break
              ;;
            *)
              echo >&2 "ERROR: Unknown LXC image $LXC_IMAGE"
              ;;
          esac
          echo "Waiting for cloud-init.."
          sleep 1s
        done
        sg lxd -c "lxc exec testcon -- apt-get update"
        sg lxd -c "lxc exec testcon -- apt-get install -y apt-transport-https gnupg2 man wget"

    - name: Copy DEB into LXC container
      run: |
        DEB_FILE=$(ls -1 *.deb)
        echo ::set-env name=DEB_FILE::$DEB_FILE
        sg lxd -c "lxc file push ${DEB_FILE} testcon/tmp/"

    - name: Install published DEB package
      if: ${{ matrix.mode == 'upgrade-from-published' }}
      run: |
        case ${MATRIX_IMAGE} in
          ubuntu:16.04) OS=ubuntu; OS_REL=xenial ;;
          ubuntu:18.04) OS=ubuntu; OS_REL=bionic ;;
          ubuntu:20.04) OS=ubuntu; OS_REL=focal ;;
          debian:9)     OS=debian; OS_REL=stretch ;;
          debian:10)    OS=debian; OS_REL=buster ;;
          *)            echo 2>&1 "ERROR: Unexpected matrix image"; exit 1 ;;
        esac
        echo "deb [arch=amd64] https://packages.nlnetlabs.nl/linux/${OS}/ ${OS_REL} main" >$HOME/nlnetlabs.list
        sg lxd -c "lxc file push $HOME/nlnetlabs.list testcon/etc/apt/sources.list.d/"
        sg lxd -c "lxc exec testcon -- wget -q https://packages.nlnetlabs.nl/aptkey.asc"
        sg lxd -c "lxc exec testcon -- apt-key add ./aptkey.asc"
        sg lxd -c "lxc exec testcon -- apt update"
        sg lxd -c "lxc exec testcon -- apt install -y krill"
      env:
        MATRIX_IMAGE: ${{ matrix.image }}

    - name: Install new DEB package
      if: ${{ matrix.mode == 'fresh-install' }}
      run: |
        sg lxd -c "lxc exec testcon -- apt-get -y install /tmp/$DEB_FILE"

    - name: Test installed packages
      run: |
        echo -e "\nKRILLC VERSION:"
        sg lxd -c "lxc exec testcon -- krillc --version"

        echo -e "\nKRILL VERSION:"
        sg lxd -c "lxc exec testcon -- krill --version"

        echo -e "\nKRILL CONF:"
        sg lxd -c "lxc exec testcon -- cat /etc/krill.conf"

        echo -e "\nKRILL DATA DIR:"
        sg lxd -c "lxc exec testcon -- ls -la /var/lib/krill"

        echo -e "\nKRILL SERVICE STATUS BEFORE ENABLE:"
        sg lxd -c "lxc exec testcon -- systemctl status krill || true"

        echo -e "\nENABLE KRILL SERVICE:"
        sg lxd -c "lxc exec testcon -- systemctl enable krill"

        echo -e "\nKRILL SERVICE STATUS AFTER ENABLE:"
        sg lxd -c "lxc exec testcon -- systemctl status krill || true"

        echo -e "\nSTART KRILL SERVICE:"
        sg lxd -c "lxc exec testcon -- systemctl start krill"
        
        echo -e "\nKRILL SERVICE STATUS AFTER START:"
        sleep 1s
        sg lxd -c "lxc exec testcon -- systemctl status krill"

        echo -e "\nKRILL MAN PAGE:"
        sg lxd -c "lxc exec testcon -- man -P cat krill"

    - name: Install new DEB package
      if: ${{ matrix.mode == 'upgrade-from-published' }}
      run: |
        sg lxd -c "lxc exec testcon -- apt-get -y install /tmp/$DEB_FILE"

    - name: Test installed packages
      if: ${{ matrix.mode == 'upgrade-from-published' }}
      run: |
        echo -e "\nKRILLC VERSION:"
        sg lxd -c "lxc exec testcon -- krillc --version"

        echo -e "\nKRILL VERSION:"
        sg lxd -c "lxc exec testcon -- krill --version"

        echo -e "\nKRILL CONF:"
        sg lxd -c "lxc exec testcon -- cat /etc/krill.conf"

        echo -e "\nKRILL DATA DIR:"
        sg lxd -c "lxc exec testcon -- ls -la /var/lib/krill"

        echo -e "\nKRILL SERVICE STATUS:"
        sg lxd -c "lxc exec testcon -- systemctl status krill || true"

        echo -e "\nKRILL MAN PAGE:"
        sg lxd -c "lxc exec testcon -- man -P cat krill"