Add an ostree-boot-complete.service to propagate staging failures

Quite a while ago we added staged deployments, which solved
a bunch of issues around the `/etc` merge.  However...a persistent
problem since then is that any failures in that process that
happened in the *previous* boot are not very visible.

We ship custom code in `rpm-ostree status` to query the previous
journal.  But that has a few problems - one is that on systems
that have been up a while, that failure message may even get
rotated out.  And second, some systems may not even have a persistent
journal at all.

A general thing we do in e.g. Fedora CoreOS testing is to check
for systemd unit failures.  We do that both in our automated tests,
and we even ship code that displays them on ssh logins.  And beyond
that obviously a lot of other projects do the same; it's easy via
`systemctl --failed`.

So to make failures more visible, change our `ostree-finalize-staged.service`
to have an internal wrapper around the process that "catches" any
errors, and copies the error message into a file in `/boot/ostree`.

Then, a new `ostree-boot-complete.service` looks for this file on
startup and re-emits the error message, and fails.

It also deletes the file.  The rationale is to avoid *continually*
warning.  For example we need to handle the case when an upgrade
process creates a new staged deployment.  Now, we could change the
ostree core code to delete the warning file when that happens instead,
but this is trying to be a conservative change.

This should make failures here much more visible as is.

src/libostree/ostree-cmdprivate.c

@@ -51,6 +51,7 @@ ostree_cmd__private__ (void)
     _ostree_repo_static_delta_delete,
     _ostree_repo_verify_bindings,
     _ostree_sysroot_finalize_staged,
+    _ostree_sysroot_boot_complete,
   };
 
   return &table;

src/libostree/ostree-cmdprivate.h

@@ -33,6 +33,7 @@ typedef struct {
   gboolean (* ostree_static_delta_delete) (OstreeRepo *repo, const char *delta_id, GCancellable *cancellable, GError **error);
   gboolean (* ostree_repo_verify_bindings) (const char *collection_id, const char *ref_name, GVariant *commit, GError **error);
   gboolean (* ostree_finalize_staged) (OstreeSysroot *sysroot, GCancellable *cancellable, GError **error);
+  gboolean (* ostree_boot_complete) (OstreeSysroot *sysroot, GCancellable *cancellable, GError **error);
 } OstreeCmdPrivateVTable;
 
 /* Note this not really "public", we just export the symbol, but not the header */

src/libostree/ostree-impl-system-generator.c

@@ -134,6 +134,8 @@ require_internal_units (const char *normal_dir,
     return FALSE;
   if (symlinkat (SYSTEM_DATA_UNIT_PATH "/ostree-finalize-staged.path", normal_dir_dfd, "multi-user.target.wants/ostree-finalize-staged.path") < 0)
     return glnx_throw_errno_prefix (error, "symlinkat");
+  if (symlinkat (SYSTEM_DATA_UNIT_PATH "/ostree-boot-complete.service", normal_dir_dfd, "multi-user.target.wants/ostree-boot-complete.service") < 0)
+    return glnx_throw_errno_prefix (error, "symlinkat");
 
   return TRUE;
 #else

src/libostree/ostree-sysroot-deploy.c

@@ -3375,10 +3375,10 @@ ostree_sysroot_stage_tree_with_options (OstreeSysroot     *self,
 }
 
 /* Invoked at shutdown time by ostree-finalize-staged.service */
-gboolean
-_ostree_sysroot_finalize_staged (OstreeSysroot *self,
-                                 GCancellable  *cancellable,
-                                 GError       **error)
+static gboolean
+_ostree_sysroot_finalize_staged_inner (OstreeSysroot *self,
+                                       GCancellable  *cancellable,
+                                       GError       **error)
 {
   /* It's totally fine if there's no staged deployment; perhaps down the line
    * though we could teach the ostree cmdline to tell systemd to activate the
@@ -3475,9 +3475,63 @@ _ostree_sysroot_finalize_staged (OstreeSysroot *self,
   if (!ostree_sysroot_prepare_cleanup (self, cancellable, error))
     return FALSE;
 
+  // Cleanup will have closed some FDs, re-ensure writability
+  if (!_ostree_sysroot_ensure_writable (self, error))
+    return FALSE;
+
   return TRUE;
 }
 
+/* Invoked at shutdown time by ostree-finalize-staged.service */
+gboolean
+_ostree_sysroot_finalize_staged (OstreeSysroot *self,
+                                 GCancellable  *cancellable,
+                                 GError       **error)
+{
+  g_autoptr(GError) finalization_error = NULL;
+  if (!_ostree_sysroot_ensure_boot_fd (self, error))
+    return FALSE;
+  if (!_ostree_sysroot_finalize_staged_inner (self, cancellable, &finalization_error))
+    {
+      g_autoptr(GError) writing_error = NULL;
+      g_assert_cmpint (self->boot_fd, !=, -1);
+      if (!glnx_file_replace_contents_at (self->boot_fd, _OSTREE_FINALIZE_STAGED_FAILURE_PATH, 
+                                           (guint8*)finalization_error->message, -1,
+                                           0, cancellable, &writing_error))
+        {
+          // We somehow failed to write the failure message...that's not great.  Maybe ENOSPC on /boot.
+          g_printerr ("Failed to write %s: %s\n", _OSTREE_FINALIZE_STAGED_FAILURE_PATH, writing_error->message);
+        }
+      g_propagate_error (error, g_steal_pointer (&finalization_error));
+      return FALSE;
+    }
+  return TRUE;
+}
+
+/* Invoked at bootup time by ostree-boot-complete.service */
+gboolean
+_ostree_sysroot_boot_complete (OstreeSysroot *self,
+                               GCancellable  *cancellable,
+                               GError       **error)
+{
+  if (!_ostree_sysroot_ensure_boot_fd (self, error))
+    return FALSE;
+
+  glnx_autofd int failure_fd = -1;
+  if (!ot_openat_ignore_enoent (self->boot_fd, _OSTREE_FINALIZE_STAGED_FAILURE_PATH, &failure_fd, error))
+    return FALSE;
+  // If we didn't find a failure log, then there's nothing to do right now.
+  // (Actually this unit shouldn't even be invoked, but we may do more in the future)
+  if (failure_fd == -1)
+    return TRUE;
+  g_autofree char *failure_data = glnx_fd_readall_utf8 (failure_fd, NULL, cancellable, error);
+  if (failure_data == NULL)
+    return glnx_prefix_error (error, "Reading from %s", _OSTREE_FINALIZE_STAGED_FAILURE_PATH);
+  // Remove the file; we don't want to continually error out.
+  (void) unlinkat (self->boot_fd, _OSTREE_FINALIZE_STAGED_FAILURE_PATH, 0);
+  return glnx_throw (error, "ostree-finalize-staged.service failed on previous boot: %s", failure_data);
+}
+
 /**
  * ostree_sysroot_deployment_set_kargs:
  * @self: Sysroot

src/libostree/ostree-sysroot-private.h

@@ -96,6 +96,9 @@ struct OstreeSysroot {
 #define _OSTREE_SYSROOT_BOOT_INITRAMFS_OVERLAYS "ostree/initramfs-overlays"
 #define _OSTREE_SYSROOT_INITRAMFS_OVERLAYS "boot/" _OSTREE_SYSROOT_BOOT_INITRAMFS_OVERLAYS
 
+// Relative to /boot, consumed by ostree-boot-complete.service
+#define _OSTREE_FINALIZE_STAGED_FAILURE_PATH "ostree/finalize-failure.stamp"
+
 gboolean
 _ostree_sysroot_ensure_writable (OstreeSysroot      *self,
                                  GError            **error);
@@ -142,6 +145,10 @@ gboolean
 _ostree_sysroot_finalize_staged (OstreeSysroot *self,
                                  GCancellable  *cancellable,
                                  GError       **error);
+gboolean
+_ostree_sysroot_boot_complete (OstreeSysroot *self,
+                               GCancellable  *cancellable,
+                               GError       **error);
 
 OstreeDeployment *
 _ostree_sysroot_deserialize_deployment_from_variant (GVariant *v,

src/libostree/ostree-sysroot.c

@@ -357,6 +357,8 @@ _ostree_sysroot_ensure_writable (OstreeSysroot      *self,
   ostree_sysroot_unload (self);
   if (!ensure_sysroot_fd (self, error))
     return FALSE;
+  if (!_ostree_sysroot_ensure_boot_fd (self, error))
+    return FALSE;
 
   return TRUE;
 }