Add greenboot-service-monitor.service for service health checking

This change adds a service to monitor whether or not specified services have
started correctly.
If enabled, the service runs after `multi-user.target` is reached,
and is required to reach `boot-complete.target`.
The monitored services can be configured in `greenboot.conf`.
By default `sshd` and `NetworkManager` are monitored.

etc/greenboot/greenboot.conf

@@ -12,4 +12,7 @@ GREENBOOT_WATCHDOG_CHECK_ENABLED=true
 ### This variable is the number of hours after an upgrade that we consider
 ### the new deployment as culprit of reboot.
 ### It has to be a positive integer. Defaults to 24 (hours).
-# GREENBOOT_WATCHDOG_GRACE_PERIOD=24
+# GREENBOOT_WATCHDOG_GRACE_PERIOD=24
+
+## Service monitor
+# GREENBOOT_MONITOR_SERVICES="sshd NetworkManager"

greenboot.spec

@@ -88,6 +88,7 @@ install -DpZm 0755 usr/lib/greenboot/check/wanted.d/* %{buildroot}%{_prefix}/lib
 %systemd_post greenboot-grub2-set-success.service
 %systemd_post greenboot-rpm-ostree-grub2-check-fallback.service
 %systemd_post redboot-auto-reboot.service
+%systemd_post greenboot-service-monitor.service
 
 %post default-health-checks
 %systemd_post greenboot-loading-message.service
@@ -102,6 +103,7 @@ install -DpZm 0755 usr/lib/greenboot/check/wanted.d/* %{buildroot}%{_prefix}/lib
 %systemd_preun greenboot-grub2-set-counter.service
 %systemd_preun greenboot-grub2-set-success.service
 %systemd_preun greenboot-rpm-ostree-grub2-check-fallback.service
+%systemd_preun greenboot-service-monitor.service
 
 %preun default-health-checks
 %systemd_preun greenboot-loading-message.service
@@ -116,6 +118,7 @@ install -DpZm 0755 usr/lib/greenboot/check/wanted.d/* %{buildroot}%{_prefix}/lib
 %systemd_postun greenboot-grub2-set-counter.service
 %systemd_postun greenboot-grub2-set-success.service
 %systemd_postun greenboot-rpm-ostree-grub2-check-fallback.service
+%systemd_postun greenboot-service-monitor.service
 
 %postun default-health-checks
 %systemd_postun greenboot-loading-message.service
@@ -156,6 +159,8 @@ install -DpZm 0755 usr/lib/greenboot/check/wanted.d/* %{buildroot}%{_prefix}/lib
 %{_unitdir}/greenboot-rpm-ostree-grub2-check-fallback.service
 %{_libexecdir}/%{name}/redboot-auto-reboot
 %{_unitdir}/redboot-auto-reboot.service
+%{_libexecdir}/%{name}/greenboot-service-monitor
+%{_unitdir}/greenboot-service-monitor.service
 
 %files default-health-checks
 %{_prefix}/lib/%{name}/check/required.d/01_repository_dns_check.sh

usr/lib/systemd/system/greenboot-service-monitor.service

@@ -0,0 +1,21 @@
+#  SPDX-License-Identifier: LGPL-2.1+
+#
+#  This file is part of greenboot.
+#
+#  greenboot is free software; you can redistribute it and/or modify it
+#  under the terms of the GNU Lesser General Public License as published by
+#  the Free Software Foundation; either version 2.1 of the License, or
+#  (at your option) any later version.
+
+[Unit]
+Description=Monitor user services
+Before=boot-complete.target
+OnFailure=redboot.target
+After=default.target graphical.target multi-user.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/libexec/greenboot/greenboot-service-monitor
+
+[Install]
+RequiredBy=boot-complete.target

usr/libexec/greenboot/greenboot-service-monitor

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -eo pipefail
+GREENBOOT_CONFIGURATION_FILE=/etc/greenboot/greenboot.conf
+service_started=true
+if test -f "$GREENBOOT_CONFIGURATION_FILE"; then
+    source $GREENBOOT_CONFIGURATION_FILE
+fi
+
+if [ -n "$GREENBOOT_MONITOR_SERVICES" ]; then
+    services=($GREENBOOT_MONITOR_SERVICES)
+else
+    echo "GREENBOOT_MONITOR_SERVICE not found, using default value"
+    services=( sshd NetworkManager )
+fi
+
+# check all the services listed
+for service in "${services[@]}"; do
+rc="$(systemctl show -p ActiveState --value $service)"
+if [[ $rc != "active" ]]; then
+  echo "<0>${service} is ${rc}"
+  service_started=false
+fi
+done
+
+# fail if any of the monitored services failed. This will make boot-complete.target fail.
+if ! $service_started ; then
+    exit 1
+fi