Move ro /sysroot bind mount of /etc into initramfs

We recently disabled the read-only /sysroot handling:
e35b82fb89

The core problem was that a lot of services run early in the
real root and want write access to things like `/var` and `/etc`.

In trying to do remounts while the system is running we introduce
too many race conditions.

Instead, just make the `/etc` bind mount in the initramfs right
after we set up the main root.  This is much more natural really,
and avoids all race conditions since nothing is running in the
sysroot yet.

The main awkward part is that since we're not linking
`ostree-prepare-root` to GLib (yet) we have a hacky parser
for the config file.  But, this is going to be fine I think.

In order to avoid parsing the config twice, pass state from
`ostree-prepare-root` to `ostree-remount` via a file in `/run`.

src/switchroot/ostree-mount-util.h

@@ -30,11 +30,13 @@
 #include <unistd.h>
 #include <fcntl.h>
 #include <string.h>
+#include <stdbool.h>
 
 #define INITRAMFS_MOUNT_VAR "/run/ostree/initramfs-mount-var"
+#define _OSTREE_SYSROOT_READONLY_STAMP "/run/ostree-sysroot-ro.stamp"
 
 static inline int
-path_is_on_readonly_fs (char *path)
+path_is_on_readonly_fs (const char *path)
 {
   struct statvfs stvfsbuf;
 

src/switchroot/ostree-prepare-root.c

@@ -60,6 +60,7 @@
 #include <sys/syscall.h>
 #include <fcntl.h>
 #include <stdio.h>
+#include <assert.h>
 #include <stdarg.h>
 #include <stdbool.h>
 #include <stdlib.h>
@@ -83,6 +84,47 @@
 /* Initialized early in main */
 static bool running_as_pid1;
 
+static inline bool
+sysroot_is_configured_ro (const char *sysroot)
+{
+  char * config_path = NULL;
+  assert (asprintf (&config_path, "%s/ostree/repo/config", sysroot) != -1);
+  FILE *f = fopen(config_path, "r");
+  if (!f)
+    {
+      fprintf (stderr, "Missing expected repo config: %s\n", config_path);
+      free (config_path);
+      return false;
+    }
+  free (config_path);
+
+  bool ret = false;
+  char *line = NULL;
+  size_t len = 0;
+  ssize_t nread;
+  /* Note getline() will reuse the previous buffer */
+  bool in_sysroot = false;
+  while ((nread = getline (&line, &len, f)) != -1)
+    {
+      /* This is an awful hack to avoid depending on GLib in the
+       * initramfs right now.
+       */
+      if (strstr (line, "[sysroot]") == line)
+        in_sysroot = true;
+      else if (*line == '[')
+        in_sysroot = false;
+      else if (in_sysroot && strstr (line, "readonly=true") == line)
+        {
+          ret = true;
+          break;
+        }
+    }
+
+  fclose (f);
+  free (line);
+  return ret;
+}
+
 static char*
 resolve_deploy_path (const char * root_mountpoint)
 {
@@ -192,6 +234,33 @@ main(int argc, char *argv[])
   if (chdir (deploy_path) < 0)
     err (EXIT_FAILURE, "failed to chdir to deploy_path");
 
+  /* Query the repository configuration - this is an operating system builder
+   * choice.  More info: https://github.com/ostreedev/ostree/pull/1767
+   */
+  const bool sysroot_readonly = sysroot_is_configured_ro (root_arg);
+  const bool sysroot_currently_writable = !path_is_on_readonly_fs (root_arg);
+
+#ifdef USE_LIBSYSTEMD
+      sd_journal_send ("MESSAGE=sysroot configured read-only: %d, currently writable: %d", 
+                      (int)sysroot_readonly, (int)sysroot_currently_writable, NULL);
+#endif
+  if (sysroot_readonly)
+    {
+      if (!sysroot_currently_writable)
+        errx (EXIT_FAILURE, "sysroot=readonly currently requires writable / in initramfs");
+      /* Now, /etc is not normally a bind mount, but if we have a readonly
+       * sysroot, we still need a writable /etc.  And to avoid race conditions
+       * we ensure it's writable in the initramfs, before we switchroot at all.
+       */
+      if (mount ("/etc", "/etc", NULL, MS_BIND, NULL) < 0)
+        err (EXIT_FAILURE, "failed to make /etc a bind mount");
+      /* Pass on the fact that we discovered a readonly sysroot to ostree-remount.service */
+      int fd = open (_OSTREE_SYSROOT_READONLY_STAMP, O_WRONLY | O_CREAT | O_CLOEXEC, 0644);
+      if (fd < 0)
+        err (EXIT_FAILURE, "failed to create %s", _OSTREE_SYSROOT_READONLY_STAMP);
+      (void) close (fd);
+    }
+
   /* Default to true, but in the systemd case, default to false because it's handled by
    * ostree-system-generator. */
   bool mount_var = true;

src/switchroot/ostree-remount.c

@@ -81,24 +81,6 @@ do_remount (const char *target,
   printf ("Remounted %s: %s\n", writable ? "rw" : "ro", target);
 }
 
-static bool
-sysroot_is_configured_ro (void)
-{
-  struct stat stbuf;
-  static const char config_path[] = "/ostree/repo/config";
-  if (stat (config_path, &stbuf) != 0)
-    return false;
-
-  g_autoptr(GKeyFile) keyfile = g_key_file_new ();
-  if (!g_key_file_load_from_file (keyfile, config_path, 0, NULL))
-    return false;
-
-  if (g_key_file_get_boolean (keyfile, "sysroot", "readonly", NULL))
-    puts ("Ignoring sysroot.readonly config; see https://github.com/coreos/fedora-coreos-tracker/issues/488.");
-
-  return false;
-}
-
 int
 main(int argc, char *argv[])
 {
@@ -124,25 +106,10 @@ main(int argc, char *argv[])
       exit (EXIT_SUCCESS);
     }
 
-  /* Query the repository configuration - this is an operating system builder
-   * choice.
-   * */
-  const bool sysroot_readonly = sysroot_is_configured_ro ();
-
-  /* Mount the sysroot read-only if we're configured to do so.
-   * Note we only get here if / is already writable.
-   */
-  do_remount ("/sysroot", !sysroot_readonly);
-
-  if (sysroot_readonly)
+  /* Handle remounting /sysroot read-only now */
+  if (unlink (_OSTREE_SYSROOT_READONLY_STAMP) == 0)
     {
-      /* Now, /etc is not normally a bind mount, but remounting the
-       * sysroot above made it read-only since it's on the same filesystem.
-       * Make it a self-bind mount, so we can then mount it read-write.
-       */
-      if (mount ("/etc", "/etc", NULL, MS_BIND, NULL) < 0)
-        err (EXIT_FAILURE, "failed to make /etc a bind mount");
-      do_remount ("/etc", true);
+      do_remount ("/sysroot", false);
     }
 
   /* If /var was created as as an OSTree default bind mount (instead of being a separate filesystem)