# GitHub Actions workflow for building and testing Krill O/S packages. Uses GitHub Actions caching to avoid rebuilding
# Rust cargo-deb , cargo generate-rpm and Krill compiled dependencies on every run.
#
# Note: at the time of writing the GH cache contents expire after a week if not used so the next build may be much
# slower as it will have to re-download/build/install lots of Rust crates.
#
# Packages are built inside Docker containers as GH Runners have extra libraries and packages installed which can cause
# package building to succeed but package installation on a real target O/S to fail, due to being built against too
# recent version of a package such as libssl or glibc.
#
# Packages are tested inside LXC/LXD containers because Docker containers don't by default support init managers such as
# systemd but we want to test systemd service unit installation and activation.

name: Packaging
on:
  push:
    branches:
    - main
    paths-ignore:
      - '.dockerignore'
      - '.github/workflow/pkg.yml'
      - 'Changelog.md'
      - 'Dockerfile'
      - 'doc/**'
      - 'docker/**'
      - 'LICENSE'
      - 'README.md'
      - 'tests/e2e/**'

  workflow_dispatch:

defaults:
  run:
    # see: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
    shell: bash --noprofile --norc -eo pipefail -x {0}

jobs:
  # Use the cargo-deb and cargo-generate-rpm Rust crates to build Debian and RPM packages respectively for installing
  # Krill.
  # See:
  #   - https://github.com/mmstick/cargo-deb
  #   - https://github.com/cat-in-136/cargo-generate-rpm
  pkg:
    strategy:
      matrix:
        pkg:
          - 'krill'
          - 'krillup'
        image:
          - 'ubuntu:xenial'   # ubuntu/16.04
          - 'ubuntu:bionic'   # ubuntu/18.04
          - 'ubuntu:focal'    # ubuntu/20.04
          - 'debian:stretch'  # debian/9
          - 'debian:buster'   # debian/10
          - 'debian:bullseye' # debian/11
          - 'centos:7'
          - 'rockylinux:8'    # compatible with EOL centos:8
        include:
          # For CentOS 7 we want to statically link with OpenSSL, but `cargo generate-rpm` doesn't build the code for
          # unlike `cargo deb` so we have to control the feature set used for building ourselves rather than in
          # `Cargo.toml`.
          - image: 'centos:7'
            extra_build_args: '--features static-openssl'

          # CentOS 8 became EOL and is in theory still usable as a build container as there is still a Docker image
          # available, and package installation can be done by switching the yum config in the container to use packages
          # from the CentOS 8 vault rather than the now offline actual CentOS 8 repository. However, due to experiencing
          # lots of timed out connections to the vault we will build the CentOS 8 compatible package in a Rocky Linux
          # container instead, as Rocky Linux is 100% compatible with CentOS 8. The server at packages.nlnetlabs.nl
          # however has a repo for CentOS 8, not Rocky Linux, and determines the repo to publish in based on the name of
          # the archive that we produce below which is in turn based by default on the container image used to build. We
          # therefore in this case need to specify that the O/S we are building for has a different name than the Docker
          # image we are building it in.
          - image: 'rockylinux:8'
            os: 'centos:8'
    env:
      CARGO_DEB_VER: 1.34.2
      CARGO_GENERATE_RPM_VER: 0.6.0
      # A Krill version of the form 'x.y.z-dev' denotes a dev build that is newer than the released x.y.z version but is
      # not yet a new release.
      NEXT_VER_LABEL: dev
    name: pkg
    runs-on: ubuntu-latest
    # Build on the oldest platform we are targeting in order to avoid https://github.com/rust-lang/rust/issues/57497.
    # Specifying container causes all of the steps in this job to run inside a Docker container.
    container: ${{ matrix.image }}

    steps:
    - name: Set vars
      id: setvars
      shell: bash
      run: |
        # Get the operating system and release name (e.g. ubuntu and xenial) from the image name (e.g. ubuntu:xenial) by
        # extracting only the parts before and after but not including the colon:
        IMAGE="${MATRIX_IMAGE}"
        if [ "${MATRIX_OS}" != "" ]; then
          IMAGE="${MATRIX_OS}"
        fi
        echo "OS_NAME=${IMAGE%:*}" >> $GITHUB_ENV
        echo "OS_REL=${IMAGE#*:}" >> $GITHUB_ENV
      env:
        MATRIX_IMAGE: ${{ matrix.image }}
        MATRIX_OS: ${{ matrix.os }}

    # Git clone the Krill code in the branch we were invoked on.
    - name: Checkout repository
      uses: actions/checkout@v2

    # Install Rust the hard way rather than using a GH Action because the action doesn't work inside a Docker container.
    - name: Install Rust
      run: |
        case ${OS_NAME} in
          debian|ubuntu)
            apt-get update
            apt-get install -y curl
            ;;
          centos)
            yum update -y
            ;;
        esac

        curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --profile minimal -y
        echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      env:
        DEBIAN_FRONTEND: noninteractive

    - name: Install compilation and other dependencies
      run: |
        case ${OS_NAME} in
          debian|ubuntu)
            apt-get install -y build-essential jq libssl-dev lintian pkg-config
            ;;
          centos)
            yum install epel-release -y
            yum update -y
            yum install -y jq openssl-devel rpmlint
            yum groupinstall -y "Development Tools"
            ;;
        esac
      env:
        DEBIAN_FRONTEND: noninteractive

    # Speed up Krill Rust builds by caching unchanged built dependencies.
    # See: https://github.com/actions/cache/blob/master/examples.md#rust---cargo
    - name: Cache Dot Cargo
      uses: actions/cache@v2
      with:
        path: |
          ~/.cargo/registry
          ~/.cargo/git
        key: ${{ matrix.image }}-${{ matrix.pkg }}-cargo-${{ hashFiles('**/Cargo.lock') }}

    # Speed up tooling installation by only re-downloading and re-building dependent crates if we change the version of
    # the tool that we are using.
    - name: Cache Cargo Deb if available
      id: cache-cargo-deb
      uses: actions/cache@v2
      with:
        path: ~/.cargo/bin/cargo-deb
        key: ${{ matrix.image }}-${{ matrix.pkg }}-cargo-deb-${{ env.CARGO_DEB_VER }}

    - name: Cache Cargo Generate RPM if available
      id: cache-cargo-generate-rpm
      uses: actions/cache@v2
      with:
        path: ~/.cargo/bin/cargo-generate-rpm
        key: ${{ matrix.image }}-${{ matrix.pkg }}-cargo-generate-rpm-${{ env.CARGO_GENERATE_RPM_VER }}

    # Only install cargo-deb or cargo-generate-rpm if not already fetched from the cache.
    - name: Install Cargo Deb if needed
      if: steps.cache-cargo-deb.outputs.cache-hit != 'true'
      run: |
        case ${OS_NAME} in
          debian|ubuntu)
            if [[ "${OS_REL}" == "xenial" ]]; then
              # Disable use of the default lzma feature which causes XZ compression to be used
              # which then causes Lintian to fail with error:
              #   E: krill: malformed-deb-archive newer compressed control.tar.xz
              # Passing --fast to cargo-deb to disable use of XZ compression didn't help.
              # See: https://github.com/kornelski/cargo-deb/issues/12
              EXTRA_CARGO_INSTALL_ARGS="--no-default-features"
            else
              EXTRA_CARGO_INSTALL_ARGS=""
            fi
            cargo install cargo-deb --version ${CARGO_DEB_VER} --locked ${EXTRA_CARGO_INSTALL_ARGS}
            ;;
        esac

    - name: Install Cargo Generate RPM if needed
      if: steps.cache-cargo-generate-rpm.outputs.cache-hit != 'true'
      run: |
        case ${OS_NAME} in
          centos)
            cargo install cargo-generate-rpm --version ${CARGO_GENERATE_RPM_VER} --locked
            ;;
        esac

    # Instruct cargo-deb or cargo-generate-rpm to build the package based on Cargo.toml settings and command line
    # arguments.
    - name: Create the package
      env:
        MATRIX_PKG: ${{ matrix.pkg }}
        EXTRA_BUILD_ARGS: ${{ matrix.extra_build_args }}
      run: |
        # Debian
        # ==============================================================================================================
        # Packages for different distributions (e.g. Stretch, Buster) of the same O/S (e.g. Debian) when served from a
        # single package repository MUST have unique package_ver_architecture triples. Cargo deb can vary the name based
        # on the 'variant' config section in use, but doesn't do so according to Debian policy (as it modifies the
        # package name, not the package version).
        #   Format: package_ver_architecture
        #   Where ver has format: [epoch:]upstream_version[-debian_revision]
        #   And debian_version should be of the form: 1<xxx>
        #   Where it is common to set <xxx> to the O/S name.
        # See:
        #   - https://unix.stackexchange.com/a/190899
        #   - https://www.debian.org/doc/debian-policy/ch-controlfields.html#version
        # Therefore we generate the version ourselves.
        #
        # In addition, Semantic Versioning and Debian version policy cannot express a pre-release label in the same way.
        # For example 0.8.0-rc.1 is a valid Cargo.toml [package].version value but when used as a Debian package version
        # 0.8.0-rc.1 would be considered _NEWER_ than the final 0.8.0 release. To express this in a Debian compatible
        # way we must replace the dash '-' with a tilda '~'.
        #
        # RPM
        # ==============================================================================================================
        # Handle the release candidate case where the version string needs to have dash replaced by tilda. The cargo
        # build command won't work if the version key in Cargo.toml contains a tilda but we have to put the tilda there
        # for when we run cargo generate-rpm so that it uses it.
        # 
        # For background on RPM versioning see:
        #   https://docs.fedoraproject.org/en-US/packaging-guidelines/Versioning/
        #
        # COMMON
        # ==============================================================================================================
        # Finally, sometimes we want a version to be NEWER than the latest release but without having to decide what
        # higher semver number to bump to. In this case we do NOT want dash '-' to become '~' because `-` is treated as
        # higher and tilda is treated as lower.
        KRILL_VER=$(cargo read-manifest | jq -r '.version')
        KRILL_NEW_VER=$(echo $KRILL_VER | tr '-' '~')
        PKG_KRILL_VER=$(echo $KRILL_NEW_VER | sed -e "s/~$NEXT_VER_LABEL/-$NEXT_VER_LABEL/")

        case ${OS_NAME} in
          debian|ubuntu)
            MAINTAINER="The NLnet Labs RPKI Team <rpki@nlnetlabs.nl>"

            # Generate the RFC 5322 format date by hand instead of using date --rfc-email because that option doesn't exist
            # on Ubuntu 16.04 and Debian 9
            RFC5322_TS=$(LC_TIME=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S %z')

            # Generate the changelog file that Debian packages are required to have.
            # See: https://www.debian.org/doc/manuals/maint-guide/dreq.en.html#changelog
            if [ ! -d target/debian ]; then
              mkdir -p target/debian
            fi
            echo "${MATRIX_PKG} (${PKG_KRILL_VER}) unstable; urgency=medium" >target/debian/changelog
            echo "  * See: https://github.com/NLnetLabs/krill/releases/tag/v${KRILL_NEW_VER}" >>target/debian/changelog
            echo " -- maintainer ${MAINTAINER}  ${RFC5322_TS}" >>target/debian/changelog

            if [ "${MATRIX_PKG}" == "krillup" ]; then
                # Ugly hack to use an alternate Cargo Deb configuration for krillup.
                sed -i -e 's/^\[package\.metadata\.deb\]/[package.metadata.moved-deb]/' \
                       -e 's/^\[package\.metadata\.krillup-deb/[package.metadata.deb/' Cargo.toml
            fi

            DEB_VER="${PKG_KRILL_VER}-1${OS_REL}"
            cargo deb --variant ${OS_NAME}-${OS_REL} --deb-version ${DEB_VER} -v -- --locked ${EXTRA_BUILD_ARGS}
            ;;
          centos)
            # Build Krill as cargo generate-rpm can't do this yet
            cargo build --release --locked -v ${EXTRA_BUILD_ARGS}

            # Strip Krill binaries as cargo generate-rpm can't do this yet
            find target/release -maxdepth 1 -type f -executable | xargs strip -s -v

            # Fix the version string to be used for the RPM package
            sed -i -e "s/$KRILL_VER/$PKG_KRILL_VER/" Cargo.toml

            # Select the correct systemd service unit file for the target operating system
            case "${OS_NAME}:${OS_REL}" in
              centos:7)
                SYSTEMD_SERVICE_UNIT_FILE="krill-ubuntu-xenial.krill.service"
                # yum install fails on older CentOS with the default LZMA compression used by cargo generate-rpm since v0.5.0
                # see: https://github.com/cat-in-136/cargo-generate-rpm/issues/30
                EXTRA_CARGO_GENERATE_RPM_ARGS="--payload-compress gzip"
                ;;
              centos:8)
                SYSTEMD_SERVICE_UNIT_FILE="krill-ubuntu-focal.krill.service"
                EXTRA_CARGO_GENERATE_RPM_ARGS=""
                ;;
              *)
                echo >&2 "ERROR: Unsupported matrix image value: '${OS_NAME}:${OS_REL}'"
                ;;
            esac

            case ${MATRIX_PKG} in
              krill)
                # Copy the chosen systemd service unit file to where Cargo.toml expects it to be
                mkdir -p target/rpm
                cp pkg/common/${SYSTEMD_SERVICE_UNIT_FILE} target/rpm/krill.service
                ;;
              krillup)
                # Ugly hack to use an alternate configuration as `cargo generate-rpm` doesn't support
                # the `cargo deb` concept of variants.
                sed -i -e 's/^\[package\.metadata\.generate-rpm/[package.metadata.moved-generate-rpm/' \
                       -e 's/^\[package\.metadata\.krillup-generate-rpm/[package.metadata.generate-rpm/' Cargo.toml
                ;;
            esac

            cargo generate-rpm ${EXTRA_CARGO_GENERATE_RPM_ARGS}
            ;;
        esac

    # See what O/S specific linting tools think of our package.
    - name: Verify the package
      run: |
        case ${OS_NAME} in
          debian|ubuntu)
            lintian -v target/debian/*.deb
            ;;
          centos)
            # cargo generate-rpm creates RPMs that rpmlint considers to have
            # errors so don't use the rpmlint exit code otherwise we will always
            # abort the workflow.
            rpmlint target/generate-rpm/*.rpm || true
            ;;
        esac

    # Upload the produced package. The artifact will be available via the GH Actions job summary and build log pages,
    # but only to users logged in to GH with sufficient rights in this project. The uploaded artifact is also downloaded
    # by the next job (see below) to sanity check that it can be installed and results in a working Krill installation.
    - name: Upload package
      uses: actions/upload-artifact@v2
      with:
        name: ${{ matrix.pkg }}_${{ env.OS_NAME }}_${{ env.OS_REL }}
        path: |
          target/debian/*.deb
          target/generate-rpm/*.rpm

  # Download and sanity check on target operating systems the packages created by previous jobs (see above). Don't test
  # on GH runners as they come with lots of software and libraries pre-installed and thus are not representative of the
  # actual deployment targets, nor do GH runners support all targets that we want to test. Don't test in Docker
  # containers as they do not support systemd.
  pkg-test:
    name: pkg-test
    needs: pkg
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        pkg:
          - 'krill'
          - 'krillup'
        image:
          - 'ubuntu:xenial'   # ubuntu/16.04
          - 'ubuntu:bionic'   # ubuntu/18.04
          - 'ubuntu:focal'    # ubuntu/20.04
          - 'debian:stretch'  # debian/9
          - 'debian:buster'   # debian/10
          - 'debian:bullseye' # debian/11
          - 'centos:7'
          - 'centos:8'
        mode:
          - 'fresh-install'
          - 'upgrade-from-published'
        # if we later add a new O/S or variant we won't have yet ever published
        # the package so can't do a test upgrade over last published version. In
        # that case add lines here like so to disable the upgrade from published
        # test for that O/S (remember to change debian:bullseye to the correct
        # O/S name!):
        #
        #
        # exclude:
        #   - image: 'debian:bullseye'
        #     mode: 'upgrade-from-published'
        exclude:
          - pkg: 'krillup'
            mode: 'upgrade-from-published'
            image: 'ubuntu:xenial'   # ubuntu/16.04
          - pkg: 'krillup'
            mode: 'upgrade-from-published'
            image: 'ubuntu:bionic'   # ubuntu/18.04
          - pkg: 'krillup'
            mode: 'upgrade-from-published'
            image: 'ubuntu:focal'    # ubuntu/20.04
          - pkg: 'krillup'
            mode: 'upgrade-from-published'
            image: 'debian:stretch'  # debian/9
          - pkg: 'krillup'
            mode: 'upgrade-from-published'
            image: 'debian:buster'   # debian/10
          - pkg: 'krillup'
            mode: 'upgrade-from-published'
            image: 'debian:bullseye' # debian/11
          - pkg: 'krillup'
            mode: 'upgrade-from-published'
            image: 'centos:7'
          - pkg: 'krillup'
            mode: 'upgrade-from-published'
            image: 'centos:8'
    steps:
    # Set some environment variables that will be available to "run" steps below in this job, and some output variables
    # that will be available in GH Action step definitions below.
    - name: Set vars
      id: setvars
      shell: bash
      run: |
        # Get the operating system and release name (e.g. ubuntu and xenial) from the image name (e.g. ubuntu:xenial) by
        # extracting only the parts before and after but not including the colon:
        OS_NAME=${MATRIX_IMAGE%:*}
        OS_REL=${MATRIX_IMAGE#*:}

        echo "OS_NAME=${OS_NAME}" >> $GITHUB_ENV
        echo "OS_REL=${OS_REL}" >> $GITHUB_ENV

        case ${MATRIX_IMAGE} in
          centos:8)
            # the CentOS 8 LXD image no longer exists since CentOS 8 hit EOL.
            # use the Rocky Linux (a CentOS 8 compatible O/S) LXD image instead.
            echo "LXC_IMAGE=images:rockylinux/8/cloud" >> $GITHUB_ENV
            ;;
          *)
            echo "LXC_IMAGE=images:${OS_NAME}/${OS_REL}/cloud" >> $GITHUB_ENV
            ;;
        esac
      env:
        MATRIX_IMAGE: ${{ matrix.image }}

    - name: Download package
      uses: actions/download-artifact@v2
      with:
        name: ${{ matrix.pkg }}_${{ env.OS_NAME }}_${{ env.OS_REL }}

    - name: Add current user to LXD group
      run: |
        sudo usermod --append --groups lxd $(whoami)

    - name: Initialize LXD
      run: |
        sudo lxd init --auto

    - name: Check LXD configuration
      run: |
        sg lxd -c "lxc info"

    # Use of IPv6 sometimes prevents yum update being able to resolve
    # mirrorlist.centos.org.
    - name: Disable LXD assignment of IPv6 addresses
      run: |
        sg lxd -c "lxc network set lxdbr0 ipv6.address none"

    - name: Launch LXC container
      run: |
        # security.nesting=true is needed to avoid error "Failed to set up mount namespacing: Permission denied" in a
        # Debian 10 container.
        sg lxd -c "lxc launch ${LXC_IMAGE} -c security.nesting=true testcon"

    # Run package update and install man and sudo support (missing in some
    # LXC/LXD O/S images) but first wait for cloud-init to finish otherwise the
    # network isn't yet ready. Don't use cloud-init status --wait as that isn't
    # supported on older O/S's like Ubuntu 16.04 and Debian 9. Use the sudo
    # package provided configuration files otherwise when using sudo we get an
    # error that the root user isn't allowed to use sudo.
    - name: Prepare container
      shell: bash
      run: |
        echo "Waiting for cloud-init.."
        while ! sudo lxc exec testcon -- ls -la /var/lib/cloud/data/result.json; do
          sleep 1s
        done

        case ${OS_NAME} in
          debian|ubuntu)
            sg lxd -c "lxc exec testcon -- apt-get update"
            sg lxd -c "lxc exec testcon -- apt-get install -y -o Dpkg::Options::=\"--force-confnew\" apt-transport-https ca-certificates man sudo wget"
            ;;
          centos)
            if [[ "${MATRIX_IMAGE}" == "centos:8" ]]; then
              # allow CentOS 8 to continue working now that it is EOL
              # see: https://stackoverflow.com/a/70930049
              sg lxd -c "lxc exec testcon -- sed -i -e 's|mirrorlist=|#mirrorlist=|g' -e 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-Linux-*"
            fi
            sg lxd -c "lxc exec testcon -- yum update -y"
            sg lxd -c "lxc exec testcon -- yum install -y man"
            ;;
        esac

    - name: Install previously published ${{ matrix.pkg }} package
      if: ${{ matrix.mode == 'upgrade-from-published' }}
      run: |
        case ${OS_NAME} in
          debian|ubuntu)
            echo "deb [arch=amd64] https://packages.nlnetlabs.nl/linux/${OS_NAME}/ ${OS_REL} main" >$HOME/nlnetlabs.list
            sg lxd -c "lxc file push $HOME/nlnetlabs.list testcon/etc/apt/sources.list.d/"
            sg lxd -c "lxc exec testcon -- wget -q https://packages.nlnetlabs.nl/aptkey.asc"
            sg lxd -c "lxc exec testcon -- apt-key add ./aptkey.asc"
            sg lxd -c "lxc exec testcon -- apt update"
            sg lxd -c "lxc exec testcon -- apt install -y ${{ matrix.pkg }}"
            ;;
          centos)
            echo '[nlnetlabs]' >$HOME/nlnetlabs.repo
            echo 'name=NLnet Labs' >>$HOME/nlnetlabs.repo
            echo 'baseurl=https://packages.nlnetlabs.nl/linux/centos/$releasever/main/$basearch' >>$HOME/nlnetlabs.repo
            echo 'enabled=1' >>$HOME/nlnetlabs.repo
            sg lxd -c "lxc file push $HOME/nlnetlabs.repo testcon/etc/yum.repos.d/"
            sg lxd -c "lxc exec testcon -- rpm --import https://packages.nlnetlabs.nl/aptkey.asc"
            sg lxd -c "lxc exec testcon -- yum install -y ${{ matrix.pkg }}"
            ;;
        esac

    - name: Copy newly built ${{ matrix.pkg }} package into the LXC container
      run: |
        case ${OS_NAME} in
          debian|ubuntu)
            DEB_FILE=$(ls -1 debian/*.deb)
            sg lxd -c "lxc file push ${DEB_FILE} testcon/tmp/"
            echo "PKG_FILE=$(basename $DEB_FILE)" >> $GITHUB_ENV
            ;;
          centos)
            RPM_FILE=$(ls -1 generate-rpm/*.rpm)
            sg lxd -c "lxc file push ${RPM_FILE} testcon/tmp/"
            echo "PKG_FILE=$(basename $RPM_FILE)" >> $GITHUB_ENV
            ;;
        esac

    - name: Install the newly built ${{ matrix.pkg }} package
      if: ${{ matrix.mode == 'fresh-install' }}
      run: |
        case ${OS_NAME} in
          debian|ubuntu)
            sg lxd -c "lxc exec testcon -- apt-get -y install /tmp/${PKG_FILE}"
            ;;
          centos)
            sg lxd -c "lxc exec testcon -- yum install -y /tmp/${PKG_FILE}"
            ;;
        esac

    - name: Test the installed krill package
      if: ${{ matrix.pkg == 'krill' }}
      run: |
        echo -e "\nKRILLC VERSION:"
        sg lxd -c "lxc exec testcon -- krillc --version"

        echo -e "\nKRILL VERSION:"
        sg lxd -c "lxc exec testcon -- krill --version"

        echo -e "\nKRILL CONF:"
        sg lxd -c "lxc exec testcon -- cat /etc/krill.conf"

        echo -e "\nKRILL DATA DIR:"
        sg lxd -c "lxc exec testcon -- ls -la /var/lib/krill"

        echo -e "\nKRILL SERVICE STATUS BEFORE ENABLE:"
        sg lxd -c "lxc exec testcon -- systemctl status krill || true"

        echo -e "\nENABLE KRILL SERVICE:"
        sg lxd -c "lxc exec testcon -- systemctl enable krill"

        echo -e "\nKRILL SERVICE STATUS AFTER ENABLE:"
        sg lxd -c "lxc exec testcon -- systemctl status krill || true"

        echo -e "\nSTART KRILL SERVICE:"
        sg lxd -c "lxc exec testcon -- systemctl start krill"
        
        echo -e "\nKRILL SERVICE STATUS AFTER START:"
        sleep 1s
        sg lxd -c "lxc exec testcon -- systemctl status krill"

        echo -e "\nKRILL MAN PAGE:"
        sg lxd -c "lxc exec testcon -- man -P cat krill"

    - name: Test the installed krillup package
      if: ${{ matrix.pkg == 'krillup' }}
      run: |
        echo -e "\nKRILLUP VERSION:"
        sg lxd -c "lxc exec testcon -- krillup --version"

        echo -e "\nKRILLUP MAN PAGE:"
        sg lxd -c "lxc exec testcon -- man -P cat krillup"

    - name: Upgrade from the published ${{ matrix.pkg }} package to the newly built package
      if: ${{ matrix.mode == 'upgrade-from-published' }}
      run: |
        case ${OS_NAME} in
          debian|ubuntu)
            sg lxd -c "lxc exec testcon -- apt-get -y install /tmp/${PKG_FILE}"
            ;;
          centos)
            sg lxd -c "lxc exec testcon -- yum install -y /tmp/${PKG_FILE}"
            ;;
        esac

    - name: Test the upgraded krill package
      if: ${{ matrix.mode == 'upgrade-from-published' && matrix.pkg == 'krill' }}
      run: |
        echo -e "\nKRILLC VERSION:"
        sg lxd -c "lxc exec testcon -- krillc --version"

        echo -e "\nKRILL VERSION:"
        sg lxd -c "lxc exec testcon -- krill --version"

        echo -e "\nKRILL CONF:"
        sg lxd -c "lxc exec testcon -- cat /etc/krill.conf"

        echo -e "\nKRILL DATA DIR:"
        sg lxd -c "lxc exec testcon -- ls -la /var/lib/krill"

        echo -e "\nKRILL SERVICE STATUS:"
        sg lxd -c "lxc exec testcon -- systemctl status krill || true"

        echo -e "\nKRILL MAN PAGE:"
        sg lxd -c "lxc exec testcon -- man -P cat krill"