nightly-build.yml

name: VyOS nightly build

on:
  schedule:
    - cron: "0 0 * * *"

  workflow_dispatch:
    inputs:
      BUILD_BY:
        description: 'Builder identifier (default: autobuild@vyos.net)'
        default: ''
      build_version:
        description: 'Version number (default: 1.5-rolling-ISO8601-TIMESTAMP)'
        default: ''
      SKIP_SMOKETEST_RAID1:
        description: 'Skip RAID1 Test'
        required: true
        type: boolean
        default: false
      SKIP_SMOKETEST_SYSTEM:
        description: 'Skip CLI smoketest'
        required: true
        type: boolean
        default: false
      SKIP_SMOKETEST_CONFIG:
        description: 'Skip config load test'
        required: true
        type: boolean
        default: false
      SKIP_SMOKETEST_TPM:
        description: 'Skip TPM test'
        required: true
        type: boolean
        default: false
      SKIP_SNAPSHOT_S3_UPLOAD:
        description: 'Skip snapshot upload to S3'
        required: true
        type: boolean
        default: false
      SKIP_RELEASE_PUBLISHING:
        description: 'Skip release publishing'
        required: true
        type: boolean
        default: false
      SKIP_SLACK_NOTIFICATIONS:
        description: 'Skip Slack notifications'
        required: true
        type: boolean
        default: false
      SKIP_DOWNLOADS_PAGE_UPDATE:
        description: 'Skip downloads page update'
        required: true
        type: boolean
        default: false

env:
  BUILD_BY: autobuild@vyos.net
  DEBIAN_MIRROR: http://deb.debian.org/debian/
  DEBIAN_SECURITY_MIRROR: http://deb.debian.org/debian-security
  VYOS_MIRROR: https://packages.vyos.net/repositories/current/

jobs:
  build_iso:
    runs-on: ubuntu-24.04
    permissions:
      contents: write
    container:
      image: vyos/vyos-build:current
      options: --sysctl net.ipv6.conf.lo.disable_ipv6=0 --privileged
    outputs:
      BUILD_BY: ${{ steps.set_env_variables.outputs.BUILD_BY }}
      build_version: ${{ steps.set_env_variables.outputs.build_version }}
      TIMESTAMP: ${{ steps.set_env_variables.outputs.TIMESTAMP }}
      PREVIOUS_SUCCESS_BUILD_TIMESTAMP: ${{ steps.set_env_variables.outputs.PREVIOUS_SUCCESS_BUILD_TIMESTAMP }}
      generic_image_iso: ${{ steps.build_generic_iso.outputs.generic_image_iso }}
      generic_image_name: ${{ steps.build_generic_iso.outputs.generic_image_name }}

    steps:
      ### Initialization ###
      - uses: actions/checkout@v4
      - name: "Initialization: set env variables"
        id: set_env_variables
        run: |
          set -x
          if [ -n "${{ github.event.inputs.BUILD_BY }}" ]; then
            echo "BUILD_BY=${{ github.event.inputs.BUILD_BY }}" >> $GITHUB_ENV
          fi
          if [ -z "${{ github.event.inputs.build_version }}" ]; then
            echo "build_version=1.5-rolling-$(date -u +%Y%m%d%H%M)" >> $GITHUB_OUTPUT
          else
            echo "build_version=${{ github.event.inputs.build_version }}" >> $GITHUB_OUTPUT
          fi
          echo "TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> $GITHUB_OUTPUT
          echo "PREVIOUS_SUCCESS_BUILD_TIMESTAMP=$(cat version.json | jq -r '.[0].timestamp')" >> $GITHUB_OUTPUT
          echo "BUILD_BY=$BUILD_BY" >> $GITHUB_OUTPUT

      - name: Clone vyos-build repo
        uses: actions/checkout@v4
        with:
          # Clone to sub-path required for minisign binary shipped in this repo
          # This might change with future Debian versions when they ship minisign
          repository: vyos/vyos-build
          path: vyos-build

      - name: Build generic ISO image
        id: build_generic_iso
        run: |
          cd vyos-build
          sudo --preserve-env ./build-vyos-image \
            --architecture amd64 \
            --build-by $BUILD_BY \
            --build-type release \
            --custom-package vyos-1x-smoketest \
            --debian-mirror $DEBIAN_MIRROR \
            --debian-security-mirror $DEBIAN_SECURITY_MIRROR \
            --version ${{ steps.set_env_variables.outputs.build_version }} \
            --vyos-mirror $VYOS_MIRROR \
            generic
          cd build
          GENERIC_IMAGE_NAME=$(jq --raw-output .artifacts[0] manifest.json | sed -e 's/.iso//')
          GENERIC_IMAGE_ISO=$(jq --raw-output .artifacts[0] manifest.json)
          echo "generic_image_iso=${GENERIC_IMAGE_ISO}" >> $GITHUB_OUTPUT
          echo "generic_image_name=${GENERIC_IMAGE_NAME}" >> $GITHUB_OUTPUT
          # Sign build artifact
          echo "${{ secrets.minisign_public_key }}" > /tmp/minisign.pub
          echo "${{ secrets.minisign_private_key }}" > /tmp/minisign.key
          echo "${{ secrets.minisign_password }}" | $GITHUB_WORKSPACE/bin/minisign -s /tmp/minisign.key -Sm ${GENERIC_IMAGE_ISO}
          $GITHUB_WORKSPACE/bin/minisign -Vm ${GENERIC_IMAGE_ISO} -x ${GENERIC_IMAGE_ISO}.minisig -p /tmp/minisign.pub
          rm -f /tmp/minisign.key /tmp/minisign.pub

      - uses: actions/upload-artifact@v4
        with:
          name: ${{ steps.build_generic_iso.outputs.generic_image_name }}
          path: |
            **/${{ steps.build_generic_iso.outputs.generic_image_iso }}
            **/${{ steps.build_generic_iso.outputs.generic_image_iso }}.minisig
          retention-days: 30
          if-no-files-found: error

  test_smoketest_cli:
    needs: build_iso
    runs-on: ubuntu-24.04
    if: ${{ !inputs.SKIP_SMOKETEST_SYSTEM }}
    container:
      image: vyos/vyos-build:current
      options: --sysctl net.ipv6.conf.lo.disable_ipv6=0 --privileged
    steps:
      # We need the test script from vyos-build repo
      - name: Clone vyos-build repo
        uses: actions/checkout@v4
        with:
          repository: vyos/vyos-build
      - uses: actions/download-artifact@v4
        with:
          name: ${{ needs.build_iso.outputs.generic_image_name }}
          path: build
      - name: VyOS CLI smoketests (no interfaces)
        id: test
        shell: bash
        run: |
          set -e
          # extract ISO image from artifact
          find build -type f -exec cp {} build \;
          ln -s ${{ needs.build_iso.outputs.generic_image_iso }} build/live-image-amd64.hybrid.iso
          sudo make test-no-interfaces | tee smoketest_make_test_no_interfaces.log
      - uses: actions/upload-artifact@v4
        with:
          name: smoketest_make_test_no_interfaces
          path: smoketest_make_test_no_interfaces.log
          retention-days: 30
          if-no-files-found: error

  test_interfaces_cli:
    needs: build_iso
    runs-on: ubuntu-24.04
    if: ${{ !inputs.SKIP_SMOKETEST_SYSTEM }}
    container:
      image: vyos/vyos-build:current
      options: --sysctl net.ipv6.conf.lo.disable_ipv6=0 --privileged
    steps:
      # We need the test script from vyos-build repo
      - name: Clone vyos-build repo
        uses: actions/checkout@v4
        with:
          repository: vyos/vyos-build
      - uses: actions/download-artifact@v4
        with:
          name: ${{ needs.build_iso.outputs.generic_image_name }}
          path: build
      - name: VyOS CLI smoketests (interfaces only)
        id: test
        shell: bash
        run: |
          set -e
          # extract ISO image from artifact
          find build -type f -exec cp {} build \;
          ln -s ${{ needs.build_iso.outputs.generic_image_iso }} build/live-image-amd64.hybrid.iso
          sudo make test-interfaces | tee smoketest_make_test_interfaces.log
      - uses: actions/upload-artifact@v4
        with:
          name: smoketest_make_test_interfaces
          path: smoketest_make_test_interfaces.log
          retention-days: 30
          if-no-files-found: error

  test_config_load:
    needs: build_iso
    runs-on: ubuntu-24.04
    if: ${{ !inputs.SKIP_SMOKETEST_CONFIG }}
    container:
      image: vyos/vyos-build:current
      options: --sysctl net.ipv6.conf.lo.disable_ipv6=0 --privileged
    steps:
      # We need the test script from vyos-build repo
      - name: Clone vyos-build repo
        uses: actions/checkout@v4
        with:
          repository: vyos/vyos-build
      - uses: actions/download-artifact@v4
        with:
          name: ${{ needs.build_iso.outputs.generic_image_name }}
          path: build
      - name: VyOS config load tests
        id: test
        shell: bash
        run: |
          set -e
          # extract ISO image from artifact
          find build -type f -exec cp {} build \;
          ln -s ${{ needs.build_iso.outputs.generic_image_iso }} build/live-image-amd64.hybrid.iso
          sudo make testc | tee smoketest_make_testc.log
      - uses: actions/upload-artifact@v4
        with:
          name: smoketest_make_testc
          path: smoketest_make_testc.log
          retention-days: 30
          if-no-files-found: error

  test_raid1_install:
    needs: build_iso
    runs-on: ubuntu-24.04
    if: ${{ !inputs.SKIP_SMOKETEST_RAID1 }}
    container:
      image: vyos/vyos-build:current
      options: --sysctl net.ipv6.conf.lo.disable_ipv6=0 --privileged
    steps:
      # We need the test script from vyos-build repo
      - name: Clone vyos-build repo
        uses: actions/checkout@v4
        with:
          repository: vyos/vyos-build
      - uses: actions/download-artifact@v4
        with:
          name: ${{ needs.build_iso.outputs.generic_image_name }}
          path: build
      - name: VyOS RAID1 installation tests
        id: test
        shell: bash
        run: |
          set -e
          # extract ISO image from artifact
          find build -type f -exec cp {} build \;
          ln -s ${{ needs.build_iso.outputs.generic_image_iso }} build/live-image-amd64.hybrid.iso
          sudo make testraid | tee smoketest_make_testraid.log
      - uses: actions/upload-artifact@v4
        with:
          name: smoketest_make_testraid
          path: smoketest_make_testraid.log
          retention-days: 30
          if-no-files-found: error

  test_encrypted_config_tpm:
    needs: build_iso
    runs-on: ubuntu-24.04
    if: ${{ !inputs.SKIP_SMOKETEST_TPM }}
    container:
      image: vyos/vyos-build:current
      options: --sysctl net.ipv6.conf.lo.disable_ipv6=0 --privileged
    steps:
      # We need the test script from vyos-build repo
      - name: Clone vyos-build repo
        uses: actions/checkout@v4
        with:
          repository: vyos/vyos-build
      - uses: actions/download-artifact@v4
        with:
          name: ${{ needs.build_iso.outputs.generic_image_name }}
          path: build
      - name: VyOS TPM encryption tests
        id: test
        shell: bash
        run: |
          set -e
          # extract ISO image from artifact
          find build -type f -exec cp {} build \;
          ln -s ${{ needs.build_iso.outputs.generic_image_iso }} build/live-image-amd64.hybrid.iso
          sudo make testtpm | tee smoketest_make_testtpm.log
      - uses: actions/upload-artifact@v4
        with:
          name: smoketest_make_testtpm
          path: smoketest_make_testtpm.log
          retention-days: 30
          if-no-files-found: error

  slack:
    needs:
      - test_config_load
      - test_raid1_install
      - test_smoketest_cli
      - test_interfaces_cli
      - test_encrypted_config_tpm
      - build_iso
    runs-on: ubuntu-24.04
    if: failure() && github.ref == 'refs/heads/current'
    steps:
      - uses: actions/download-artifact@v4
        with:
          path: logs
          pattern: smoketest_make_*
      - name: "Slack notification: filter Smoketest failed logs"
        id: filter_smoketest_failed_logs
        run: |
          for logfile in $(find logs -type f -name "*.log")
          do
              echo $logfile
              cat $logfile | grep '... FAIL' >> ${{ github.workspace }}/smoketest_filtered.log || true
              cat $logfile | grep 'ERROR -' >> ${{ github.workspace }}/smoketest_filtered.log || true
              cat $logfile | grep 'Error:' -A12 >> ${{ github.workspace }}/smoketest_filtered.log || true
          done
          ls -al

      - name: "Slack notification: send to channel"
        id: send_a_slack_failure_notification
        uses: slackapi/slack-github-action@v1.25.0
        with:
          payload: |
            {
              "text": "*VyOS nightly build automation did not succeed*",
              "attachments": [
                {
                  "color": "FF6600",
                  "fields": [
                    {
                      "title": "Repository",
                      "short": true,
                      "value": "${{ github.repository }}"
                    },
                    {
                      "title": "Workflow",
                      "short": true,
                      "value": "${{ github.workflow }}"
                    },
                    {
                      "title": "Trigger",
                      "short": true,
                      "value": "${{ github.event_name }}"
                    },
                    {
                      "title": "Run number",
                      "short": true,
                      "value": "${{ github.run_number }}"
                    },
                    {
                      "title": "URL",
                      "short": false,
                      "value": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
                    }
                  ]
                }
              ]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

      - name: "Slack notification: send Smoketest log excerpt"
        uses: MeilCli/slack-upload-file@v4
        with:
          slack_token: ${{ secrets.SLACK_BOT_TOKEN }}
          channel_id: ${{ secrets.SLACK_CHANNEL_ID }}
          file_type: 'text'
          file_path: ${{ github.workspace }}/smoketest_filtered.log
          initial_comment: Smoketest log excerpt
          if_no_files_found: ignore

  publish:
    needs:
      - test_config_load
      - test_raid1_install
      - test_smoketest_cli
      - test_interfaces_cli
      - test_encrypted_config_tpm
      - build_iso
    runs-on: ubuntu-24.04
    permissions:
      contents: write
    if: success() && !inputs.SKIP_RELEASE_PUBLISHING && github.ref == 'refs/heads/current'
    steps:
      - uses: actions/checkout@v4
      - name: Clone vyos-build repo
        uses: actions/checkout@v4
        with:
          repository: vyos/vyos-build
          path: vyos-build
      - name: Clone vyos-1x source code
        uses: actions/checkout@v4
        with:
          repository: vyos/vyos-1x
          path: vyos-1x
      -  id: commit_ids
         shell: bash
         run: |
          cd vyos-build
          echo "CHANGELOG_COMMIT_VYOS_BUILD=$(git log --since "${{ needs.build_iso.outputs.PREVIOUS_SUCCESS_BUILD_TIMESTAMP }}" --format="%H" --reverse | head -n1)" >> $GITHUB_OUTPUT
          cd ../vyos-1x
          echo "CHANGELOG_COMMIT_VYOS_1X=$(git log --since "${{ needs.build_iso.outputs.PREVIOUS_SUCCESS_BUILD_TIMESTAMP }}" --format="%H" --reverse | head -n1)" >> $GITHUB_OUTPUT

      - name: "Release publishing: generate changelog for vyos-1x"
        id: generate_changelog_for_vyos-1x
        uses: mikepenz/release-changelog-builder-action@v4.1.0
        with:
          owner: "vyos"
          repo: "vyos-1x"
          fetchReviewers: false
          fromTag: ${{ steps.commit_ids.outputs.CHANGELOG_COMMIT_VYOS_1X }}
          toTag: HEAD
          configurationJson: |
            {
              "categories": [{"title": "", "labels": []}],
              "template": "#{{CHANGELOG}}",
              "pr_template": "- #{{TITLE}}\n   - PR: vyos/vyos-1x##{{NUMBER}}"
            }

      - name: "Release publishing: generate changelog for vyos-build"
        id: generate_changelog_for_vyos-build
        uses: mikepenz/release-changelog-builder-action@v4.1.0
        with:
          owner: "vyos"
          repo: "vyos-build"
          fetchReviewers: false
          fromTag: ${{ steps.commit_ids.outputs.CHANGELOG_COMMIT_VYOS_BUILD }}
          toTag: HEAD
          configurationJson: |
            {
              "categories": [{"title": "", "labels": []}],
              "template": "#{{CHANGELOG}}",
              "pr_template": "- #{{TITLE}}\n   - PR: vyos/vyos-build##{{NUMBER}}"
            }

      - name: "Release publishing: generate CHANGELOG.md"
        run: |
          cat <<EOF > CHANGELOG.md
          ## vyos-1x
          ${{ steps.generate_changelog_for_vyos-1x.outputs.changelog }}
          ## vyos-build
          ${{ steps.generate_changelog_for_vyos-build.outputs.changelog }}
          EOF
          cat CHANGELOG.md

      - name: "Release publishing: create version.json"
        uses: jsdaniell/create-json@v1.2.3
        with:
          name: "version.json"
          json: |
            [
              {
                "url": "https://github.com/vyos/vyos-nightly-build/releases/download/${{ needs.build_iso.outputs.build_version }}/${{ needs.build_iso.outputs.generic_image_iso }}",
                "version": "${{ needs.build_iso.outputs.build_version }}",
                "timestamp": "${{ needs.build_iso.outputs.TIMESTAMP }}"
              }
            ]

      - name: "Release publishing: check if the repository was modified during runtime to prevent autocommit failure"
        id: check_if_the_repository_was_modified_during_runtime
        run: |
          sudo chown -R $(whoami):$(whoami) ./*
          git pull --autostash --rebase

      - name: "Release publishing: create autocommit and tag"
        uses: stefanzweifel/git-auto-commit-action@v5
        with:
          tagging_message: ${{ needs.build_iso.outputs.build_version }}
          commit_message: ${{ needs.build_iso.outputs.build_version }}
          commit_author: "vyosbot <${{ needs.build_iso.outputs.BUILD_BY }}>"

      - uses: actions/download-artifact@v4
        with:
          name: ${{ needs.build_iso.outputs.generic_image_name }}

      - name: Debug artifact path
        run: |
          find . -type f

      - name: "Release publishing: publish release"
        uses: softprops/action-gh-release@v2
        with:
          body_path: CHANGELOG.md
          tag_name: ${{ needs.build_iso.outputs.build_version }}
          fail_on_unmatched_files: true
          files: |
            ./${{ needs.build_iso.outputs.generic_image_iso }}
            ./${{ needs.build_iso.outputs.generic_image_iso }}.minisig

      - uses: Nats-ji/delete-old-releases@v1.0.1
        with:
          token: ${{ secrets.CUSTOM_GITHUB_TOKEN }}
          keep-count: 30
          keep-old-minor-releases: false

  update_download_page:
    needs:
      - publish
    uses: vyos/community.vyos.net/.github/workflows/main.yml@production
    with:
      branch: production
      aws_amplify_region: ${{ vars.AWS_AMPLIFY_REGION }}
      aws_amplify_app_id: ${{ vars.AWS_AMPLIFY_APP_ID }}
    secrets:
      AWS_AMPLIFY_ACCESS_KEY_ID: ${{ secrets.AWS_AMPLIFY_ACCESS_KEY_ID }}
      AWS_AMPLIFY_SECRET_ACCESS_KEY: ${{ secrets.AWS_AMPLIFY_SECRET_ACCESS_KEY }}