Merge pull request #28 from LorbusChris/devel

Update to v0.9

greenboot.spec

@@ -1,5 +1,5 @@
 Name:               greenboot
-Version:            0.8
+Version:            0.9
 Release:            1%{?dist}
 Summary:            Generic Health Check Framework for systemd
 License:            LGPLv2+
@@ -67,16 +67,15 @@ Requires:           %{name} = %{version}-%{release}
 %build
 
 %install
+mkdir -p %{buildroot}%{_exec_prefix}/lib/motd.d/
+mkdir -p %{buildroot}%{_libexecdir}/%{name}
 mkdir -p %{buildroot}%{_sysconfdir}/%{name}/check/required.d
 mkdir    %{buildroot}%{_sysconfdir}/%{name}/check/wanted.d
 mkdir    %{buildroot}%{_sysconfdir}/%{name}/green.d
 mkdir    %{buildroot}%{_sysconfdir}/%{name}/red.d
 mkdir -p %{buildroot}%{_unitdir}
-install -DpZm 0755 usr/libexec/greenboot/greenboot %{buildroot}%{_libexecdir}/%{name}/%{name}
-install -DpZm 0755 usr/libexec/greenboot/greenboot-grub2-set-counter %{buildroot}%{_libexecdir}/%{name}/greenboot-grub2-set-counter
-install -DpZm 0755 usr/libexec/greenboot/greenboot-rpm-ostree-grub2-check-fallback %{buildroot}%{_libexecdir}/%{name}/greenboot-rpm-ostree-grub2-check-fallback
-install -DpZm 0755 usr/libexec/greenboot/greenboot-status %{buildroot}%{_libexecdir}/%{name}/greenboot-status
-install -DpZm 0755 usr/libexec/greenboot/redboot-auto-reboot-check %{buildroot}%{_libexecdir}/%{name}/redboot-auto-reboot-check
+mkdir -p %{buildroot}%{_tmpfilesdir}
+install -DpZm 0755 usr/libexec/greenboot/* %{buildroot}%{_libexecdir}/%{name}
 install -DpZm 0644 usr/lib/motd.d/boot-status %{buildroot}%{_exec_prefix}/lib/motd.d/boot-status
 install -DpZm 0644 usr/lib/systemd/system/* %{buildroot}%{_unitdir}
 install -DpZm 0644 usr/lib/tmpfiles.d/greenboot-status-motd.conf %{buildroot}%{_tmpfilesdir}/greenboot-status-motd.conf
@@ -86,6 +85,7 @@ install -DpZm 0755 etc/greenboot/check/wanted.d/* %{buildroot}%{_sysconfdir}/%{n
 %post
 %systemd_post greenboot-healthcheck.service
 %systemd_post greenboot.service
+%systemd_post greenboot.target
 %systemd_post redboot.service
 %systemd_post redboot.target
 
@@ -121,6 +121,7 @@ install -DpZm 0755 etc/greenboot/check/wanted.d/* %{buildroot}%{_sysconfdir}/%{n
 %postun
 %systemd_postun greenboot-healthcheck.service
 %systemd_postun greenboot.service
+%systemd_postun greenboot.target
 %systemd_postun redboot.service
 %systemd_postun redboot.target
 
@@ -141,6 +142,7 @@ install -DpZm 0755 etc/greenboot/check/wanted.d/* %{buildroot}%{_sysconfdir}/%{n
 %{_libexecdir}/%{name}/%{name}
 %{_unitdir}/greenboot-healthcheck.service
 %{_unitdir}/greenboot.service
+%{_unitdir}/greenboot.target
 %{_unitdir}/redboot.service
 %{_unitdir}/redboot.target
 %dir %{_sysconfdir}/%{name}
@@ -168,10 +170,13 @@ install -DpZm 0755 etc/greenboot/check/wanted.d/* %{buildroot}%{_sysconfdir}/%{n
 %{_unitdir}/greenboot-grub2-set-counter.service
 
 %files reboot
-%{_libexecdir}/%{name}/redboot-auto-reboot-check
+%{_libexecdir}/%{name}/redboot-auto-reboot
 %{_unitdir}/redboot-auto-reboot.service
 
 %changelog
+* Fri Apr 03 2020 Christian Glombek <[email protected]> - 0.9-1
+- Update to v0.9
+
 * Wed Feb 05 2020 Christian Glombek <[email protected]> - 0.8-1
 - Update to v0.8
 - Add guard against bootlooping in redboot-auto-reboot.service

usr/lib/systemd/system/greenboot-grub2-set-counter.service

@@ -9,13 +9,14 @@
 
 [Unit]
 Description=Set grub2 boot counter in preparation of upgrade
+DefaultDependencies=no
 Before=ostree-finalize-staged.service
 Conflicts=greenboot-grub2-set-success.service
 
 [Service]
 Type=oneshot
 RemainAfterExit=yes
-ExecStop=/usr/libexec/greenboot/greenboot-grub2-set-counter 3
+ExecStart=/usr/libexec/greenboot/greenboot-grub2-set-counter 3
 
 [Install]
 RequiredBy=ostree-finalize-staged.service

usr/lib/systemd/system/greenboot.service

@@ -9,10 +9,8 @@
 
 [Unit]
 Description=greenboot Success Scripts Runner
-Requires=boot-complete.target
-After=boot-complete.target
-OnFailure=redboot.target
-OnFailureJobMode=fail
+Requires=greenboot.target
+After=greenboot.target
 
 [Service]
 Type=oneshot

usr/lib/systemd/system/greenboot.target

@@ -0,0 +1,18 @@
+#  SPDX-License-Identifier: LGPL-2.1+
+#
+#  This file is part of greenboot.
+#
+#  greenboot is free software; you can redistribute it and/or modify it
+#  under the terms of the GNU Lesser General Public License as published by
+#  the Free Software Foundation; either version 2.1 of the License, or
+#  (at your option) any later version.
+
+[Unit]
+Description=greenboot Successful Boot Target
+Requires=boot-complete.target
+After=boot-complete.target
+OnFailure=redboot.target
+OnFailureJobMode=fail
+
+[Install]
+WantedBy=multi-user.target

usr/lib/systemd/system/redboot-auto-reboot.service

@@ -12,12 +12,10 @@ Description=Reboot on red boot status
 DefaultDependencies=no
 After=redboot.service
 Wants=redboot.service
-SuccessAction=reboot
 
 [Service]
 Type=oneshot
-RemainAfterExit=yes
-ExecStart=/usr/libexec/greenboot/redboot-auto-reboot-check
+ExecStart=/usr/libexec/greenboot/redboot-auto-reboot
 
 [Install]
 WantedBy=redboot.target

usr/lib/systemd/system/redboot.target

@@ -10,3 +10,4 @@
 [Unit]
 Description=Generic red boot target
 Conflicts=boot-complete.target
+Conflicts=greenboot.target

usr/libexec/greenboot/greenboot

@@ -1,17 +1,19 @@
 #!/bin/bash
 set -euo pipefail
 IFS=$'\n\t'
+LC_ALL=C
 
 script_runner () {
   local scripts_dir=$1; shift
   local mode=$1; shift
   local start_msg=$1; shift
   echo "$start_msg"
-  for script in $(find $scripts_dir -name '*.sh'); do
+  for script in $(find "$scripts_dir" -name '*.sh' | sort); do
     local rc=0
-    systemd-cat -t "$(basename $script)" bash $script || rc=$?
+    systemd-cat -t "$(basename "$script")" bash "$script" || rc=$?
     if [ $rc -ne 0 ]; then
-      local failure_msg="Script '$(basename $script)' FAILURE (exit code '$rc')"
+      local failure_msg
+      failure_msg="Script '$(basename "$script")' FAILURE (exit code '$rc')"
       case "$mode" in
         "relaxed")
           echo "<2>$failure_msg. Continuing..." >&2
@@ -21,7 +23,7 @@ script_runner () {
           exit $rc
       esac
     else
-      echo "<6>Script '$(basename $script)' SUCCESS"
+      echo "<6>Script '$(basename "$script")' SUCCESS"
     fi
   done
 }

usr/libexec/greenboot/greenboot-grub2-set-counter

@@ -2,7 +2,7 @@
 set -euo pipefail
 
 max_boot_attempts=${1:-3} # default to 3 attempts
-grub2-editenv - set boot_counter=$max_boot_attempts
+grub2-editenv - set boot_counter="$max_boot_attempts"
 grub2-editenv - set boot_success=0
 
 echo "<3>GRUB2 environment variables have been set for system upgrade"

usr/libexec/greenboot/greenboot-status

@@ -4,15 +4,22 @@ IFS=$'\n\t'
 
 status=""
 
+# healthcheck logs from current boot
+healthcheck_info="$(journalctl -u greenboot-healthcheck.service -p 2 -b -0 -o cat)"||""
+if [ -n "$healthcheck_info" ]; then
+    status+="$healthcheck_info"$'\n'
+fi
+
 if [ "$(systemctl is-active boot-complete.target)" = "active" ]; then
-    # greenboot logs from current boot
+    # greenboot logs (post healthchecks) from current boot
     status+="$(journalctl -u greenboot.service -p 5 -b -0 -o cat)"$'\n'
 else
-    # redboot logs from current boot
-    status+="$(journalctl -u redboot.service -p 0 -b -0 -o cat)"$'\n'
-    # healthcheck logs from current boot
-    status+="$(journalctl -u greenboot-healthcheck.service -p 2 -b -0 -o cat)"$'\n'
-    # redboot-auto-reboot.service logs from current boot
+    # redboot logs (post healthchecks) from current boot
+    redboot_info+="$(journalctl -u redboot.service -p 0 -b -0 -o cat)"||""
+    if [ -n "$redboot_info" ]; then
+        status+="$redboot_info"$'\n'
+    fi
+    # redboot-auto-reboot.service logs (post redboot.service) from current boot
     reboot_info="$(journalctl -u redboot-auto-reboot -p 1 -b -0 -o cat)"||""
     if [ -n "$reboot_info" ]; then
         status+="$reboot_info"

usr/libexec/greenboot/redboot-auto-reboot

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -euo pipefail
+
+bootloader_entries=(/boot/loader/entries/*)
+bootloader_entry_count=${#bootloader_entries[@]}
+
+# If boot_counter is unset, do nothing
+if ! grub2-editenv list | grep -q "^boot_counter="; then
+  echo "<0>SYSTEM is UNHEALTHY, but boot_counter is unset in grubenv. Manual intervention necessary."
+  exit 1
+# In case we've booted into our fallback deployment and still don't reach boot-complete.target, do nothing.
+elif grub2-editenv list | grep -q "^boot_counter=-1$"; then
+  echo "<0>FALLBACK BOOT DETECTED but SYSTEM is still UNHEALTHY. Manual intervention necessary."
+  exit 2
+# If we have only one (or less) bootloader entries, don't reboot
+elif [ "$bootloader_entry_count" -le 1 ]; then
+  echo "<0>SYSTEM is UNHEALTHY, but bootlader entry count is $bootloader_entry_count. Manual intervention necessary."
+  exit 3
+fi
+
+echo "<1>SYSTEM is UNHEALTHY. Rebooting..."
+reboot