sysroot: Handle ro /boot but rw /sysroot

The recent change in https://github.com/coreos/fedora-coreos-config/pull/659
broke some of our tests that do `mount -o remount,rw /sysroot` but
leave `/boot` read-only.

We had code for having `/boot` read-only before `/sysroot` but
in practice we had a file descriptor for `/sysroot` that we opened
before the remount that would happen later on.

Clean things up here so that in the library, we also remount
`/boot` at the same time we remount `/sysroot` if either are readonly.

Delete the legacy code for remounting `/boot` rw if we're not in
a mount namespace.  I am fairly confident most users are either
using the `ostree` CLI, or they're using the mount namespace.

src/libostree/ostree-sysroot.c

@@ -295,6 +295,31 @@ _ostree_sysroot_ensure_boot_fd (OstreeSysroot *self, GError **error)
   return TRUE;
 }
 
+static gboolean
+remount_writable (const char *path, gboolean *did_remount, GError **error)
+{
+  *did_remount = FALSE;
+  struct statvfs stvfsbuf;
+  if (statvfs (path, &stvfsbuf) < 0)
+    {
+      if (errno != ENOENT)
+        return glnx_throw_errno_prefix (error, "statvfs(%s)", path);
+      else
+        return TRUE;
+    }
+
+  if ((stvfsbuf.f_flag & ST_RDONLY) != 0)
+    {
+      /* OK, let's remount writable. */
+      if (mount (path, path, NULL, MS_REMOUNT | MS_RELATIME, "") < 0)
+        return glnx_throw_errno_prefix (error, "Remounting %s read-write", path);
+      *did_remount = TRUE;
+      g_debug ("remounted %s writable", path);
+    }
+
+  return TRUE;
+}
+
 /* Remount /sysroot read-write if necessary */
 gboolean
 _ostree_sysroot_ensure_writable (OstreeSysroot      *self,
@@ -314,19 +339,19 @@ _ostree_sysroot_ensure_writable (OstreeSysroot      *self,
   if (!self->root_is_ostree_booted)
     return TRUE;
 
-  /* Check if /sysroot is a read-only mountpoint */
-  struct statvfs stvfsbuf;
-  if (statvfs ("/sysroot", &stvfsbuf) < 0)
-    return glnx_throw_errno_prefix (error, "fstatvfs(/sysroot)");
-  if ((stvfsbuf.f_flag & ST_RDONLY) == 0)
-    return TRUE;
+  /* In these cases we also require /boot */
+  if (!_ostree_sysroot_ensure_boot_fd (self, error))
+    return FALSE;
 
-  /* OK, let's remount writable. */
-  if (mount ("/sysroot", "/sysroot", NULL, MS_REMOUNT | MS_RELATIME, "") < 0)
-    return glnx_throw_errno_prefix (error, "Remounting /sysroot read-write");
+  gboolean did_remount_sysroot = FALSE;
+  if (!remount_writable ("/sysroot", &did_remount_sysroot, error))
+    return FALSE;
+  gboolean did_remount_boot = FALSE;
+  if (!remount_writable ("/boot", &did_remount_boot, error))
+    return FALSE;
 
-  /* Reopen our fd */
-  glnx_close_fd (&self->sysroot_fd);
+  /* Now close and reopen our file descriptors */
+  ostree_sysroot_unload (self);
   if (!ensure_sysroot_fd (self, error))
     return FALSE;