Add support for pulling from remote archives

This necessitated a large set of changes.

We now support an "archive" mode for repositories.  In this mode,
files are stored "packed" rather than hard linked.  This allows one to
e.g. store an OSTree repository with root-owned files as non-root.  It
is also used as the basis for serving repositories via HTTP.

While doing this I realized that GVariant is endianness-dependent; I
decided to just store all data in big endian.

src/libostree/ostree-core.c

@@ -27,13 +27,27 @@
 #include <sys/types.h>
 #include <attr/xattr.h>
 
-static char *
-stat_to_string (struct stat *stbuf)
+gboolean
+ostree_validate_checksum_string (const char *sha256,
+                                 GError    **error)
 {
-  return g_strdup_printf ("%u:%u:%u",
-                          (guint32)(stbuf->st_mode & ~S_IFMT),
-                          (guint32)stbuf->st_uid, 
-                          (guint32)stbuf->st_gid);
+  if (strlen (sha256) != 64)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Invalid rev '%s'", sha256);
+      return FALSE;
+    }
+  return TRUE;
+}
+
+
+void
+ostree_checksum_update_stat (GChecksum *checksum, guint32 uid, guint32 gid, guint32 mode)
+{
+  guint32 perms = (mode & ~S_IFMT);
+  g_checksum_update (checksum, (guint8*) &uid, 4);
+  g_checksum_update (checksum, (guint8*) &gid, 4);
+  g_checksum_update (checksum, (guint8*) &perms, 4);
 }
 
 static char *
@@ -144,6 +158,7 @@ ostree_get_xattrs_for_path (const char *path,
     }
 
   ret = g_variant_builder_end (&builder);
+  g_variant_ref_sink (ret);
  out:
   if (!ret)
     g_variant_builder_clear (&builder);
@@ -154,9 +169,10 @@ ostree_get_xattrs_for_path (const char *path,
 
 gboolean
 ostree_stat_and_checksum_file (int dir_fd, const char *path,
-                                 GChecksum **out_checksum,
-                                 struct stat *out_stbuf,
-                                 GError **error)
+                               OstreeObjectType objtype,
+                               GChecksum **out_checksum,
+                               struct stat *out_stbuf,
+                               GError **error)
 {
   GChecksum *content_sha256 = NULL;
   GChecksum *content_and_meta_sha256 = NULL;
@@ -202,10 +218,12 @@ ostree_stat_and_checksum_file (int dir_fd, const char *path,
         }
     }
 
-  stat_string = stat_to_string (&stbuf);
-  xattrs = ostree_get_xattrs_for_path (path, error);
-  if (!xattrs)
-    goto out;
+  if (objtype == OSTREE_OBJECT_TYPE_FILE)
+    {
+      xattrs = ostree_get_xattrs_for_path (path, error);
+      if (!xattrs)
+        goto out;
+    }
 
   content_sha256 = g_checksum_new (G_CHECKSUM_SHA256);
  
@@ -224,6 +242,8 @@ ostree_stat_and_checksum_file (int dir_fd, const char *path,
   else if (S_ISLNK(stbuf.st_mode))
     {
       symlink_target = g_malloc (PATH_MAX);
+
+      g_assert (objtype == OSTREE_OBJECT_TYPE_FILE);
       
       bytes_read = readlinkat (dir_fd, basename, symlink_target, PATH_MAX);
       if (bytes_read < 0)
@@ -235,6 +255,7 @@ ostree_stat_and_checksum_file (int dir_fd, const char *path,
     }
   else if (S_ISCHR(stbuf.st_mode) || S_ISBLK(stbuf.st_mode))
     {
+      g_assert (objtype == OSTREE_OBJECT_TYPE_FILE);
       device_id = g_strdup_printf ("%u", (guint)stbuf.st_rdev);
       g_checksum_update (content_sha256, (guint8*)device_id, strlen (device_id));
     }
@@ -249,8 +270,12 @@ ostree_stat_and_checksum_file (int dir_fd, const char *path,
 
   content_and_meta_sha256 = g_checksum_copy (content_sha256);
 
-  g_checksum_update (content_and_meta_sha256, (guint8*)stat_string, strlen (stat_string));
-  g_checksum_update (content_and_meta_sha256, (guint8*)g_variant_get_data (xattrs), g_variant_get_size (xattrs));
+  if (objtype == OSTREE_OBJECT_TYPE_FILE)
+    {
+      ostree_checksum_update_stat (content_and_meta_sha256, stbuf.st_uid,
+                                   stbuf.st_gid, stbuf.st_mode);
+      g_checksum_update (content_and_meta_sha256, (guint8*)g_variant_get_data (xattrs), g_variant_get_size (xattrs));
+    }
 
   *out_stbuf = stbuf;
   *out_checksum = content_and_meta_sha256;
@@ -267,6 +292,264 @@ ostree_stat_and_checksum_file (int dir_fd, const char *path,
     g_variant_unref (xattrs);
   if (content_sha256)
     g_checksum_free (content_sha256);
-
   return ret;
 }
+
+gboolean
+ostree_set_xattrs (const char *path, GVariant *xattrs, GError **error)
+{
+  gboolean ret = FALSE;
+  int i, n;
+
+  n = g_variant_n_children (xattrs);
+  for (i = 0; i < n; i++)
+    {
+      const guint8* name;
+      GVariant *value;
+      const guint8* value_data;
+      gsize value_len;
+      gboolean loop_err;
+
+      g_variant_get_child (xattrs, i, "(^&ay@ay)",
+                           &name, &value);
+      value_data = g_variant_get_fixed_array (value, &value_len, 1);
+      
+      loop_err = lsetxattr (path, (char*)name, (char*)value_data, value_len, XATTR_REPLACE) < 0;
+      
+      g_variant_unref (value);
+      if (loop_err)
+        {
+          ot_util_set_error_from_errno (error, errno);
+          goto out;
+        }
+    }
+
+  ret = TRUE;
+ out:
+  return ret;
+}
+
+gboolean
+ostree_parse_metadata_file (const char                  *path,
+                            OstreeSerializedVariantType *out_type,
+                            GVariant                   **out_variant,
+                            GError                     **error)
+{
+  GMappedFile *mfile = NULL;
+  gboolean ret = FALSE;
+  GVariant *ret_variant = NULL;
+  GVariant *container = NULL;
+  guint32 ret_type;
+
+  mfile = g_mapped_file_new (path, FALSE, error);
+  if (mfile == NULL)
+    {
+      goto out;
+    }
+  else
+    {
+      container = g_variant_new_from_data (G_VARIANT_TYPE (OSTREE_SERIALIZED_VARIANT_FORMAT),
+                                           g_mapped_file_get_contents (mfile),
+                                           g_mapped_file_get_length (mfile),
+                                           FALSE,
+                                           (GDestroyNotify) g_mapped_file_unref,
+                                           mfile);
+      g_variant_get (container, "(uv)",
+                     &ret_type, &ret_variant);
+      if (ret_type <= 0 || ret_type > OSTREE_SERIALIZED_VARIANT_LAST)
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "Corrupted metadata object '%s'; invalid type %d", path, ret_type);
+          goto out;
+        }
+      mfile = NULL;
+    }
+
+  ret = TRUE;
+  *out_type = ret_type;
+  *out_variant = ret_variant;
+  ret_variant = NULL;
+ out:
+  if (ret_variant)
+    g_variant_unref (ret_variant);
+  if (container != NULL)
+    g_variant_unref (container);
+  if (mfile != NULL)
+    g_mapped_file_unref (mfile);
+  return ret;
+}
+
+char *
+ostree_get_relative_object_path (const char *checksum,
+                                 OstreeObjectType type,
+                                 gboolean         archive)
+{
+  GString *path;
+  const char *type_string;
+
+  g_assert (strlen (checksum) == 64);
+
+  path = g_string_new ("objects/");
+
+  g_string_append_len (path, checksum, 2);
+  g_string_append_c (path, '/');
+  g_string_append (path, checksum + 2);
+  switch (type)
+    {
+    case OSTREE_OBJECT_TYPE_FILE:
+      if (archive)
+        type_string = ".packfile";
+      else
+        type_string = ".file";
+      break;
+    case OSTREE_OBJECT_TYPE_META:
+      type_string = ".meta";
+      break;
+    default:
+      g_assert_not_reached ();
+    }
+  g_string_append (path, type_string);
+  return g_string_free (path, FALSE);
+}
+
+gboolean
+ostree_pack_object (GOutputStream     *output,
+                    GFile             *file,
+                    OstreeObjectType  objtype,
+                    GCancellable     *cancellable,
+                    GError          **error)
+{
+  gboolean ret = FALSE;
+  char *path = NULL;
+  GFileInfo *finfo = NULL;
+  GFileInputStream *instream = NULL;
+  gboolean pack_builder_initialized = FALSE;
+  GVariantBuilder pack_builder;
+  GVariant *pack_variant = NULL;
+  GVariant *xattrs = NULL;
+  gsize bytes_written;
+
+  path = g_file_get_path (file);
+
+  finfo = g_file_query_info (file, "standard::type,standard::size,standard::is-symlink,standard::symlink-target,unix::*",
+                             G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, cancellable, error);
+  if (!finfo)
+    goto out;
+
+  if (objtype == OSTREE_OBJECT_TYPE_META)
+    {
+      guint64 object_size_be = GUINT64_TO_BE ((guint64)g_file_info_get_size (finfo));
+      if (!g_output_stream_write_all (output, &object_size_be, 8, &bytes_written, cancellable, error))
+        goto out;
+
+      instream = g_file_read (file, NULL, error);
+      if (!instream)
+        goto out;
+      
+      if (g_output_stream_splice (output, (GInputStream*)instream, 0, cancellable, error) < 0)
+        goto out;
+    }
+  else
+    {
+      guint32 uid, gid, mode;
+      guint32 device = 0;
+      guint32 metadata_size_be;
+      const char *target = NULL;
+      guint64 object_size;
+
+      uid = g_file_info_get_attribute_uint32 (finfo, G_FILE_ATTRIBUTE_UNIX_UID);
+      gid = g_file_info_get_attribute_uint32 (finfo, G_FILE_ATTRIBUTE_UNIX_GID);
+      mode = g_file_info_get_attribute_uint32 (finfo, G_FILE_ATTRIBUTE_UNIX_MODE);
+
+      g_variant_builder_init (&pack_builder, G_VARIANT_TYPE (OSTREE_PACK_FILE_VARIANT_FORMAT));
+      pack_builder_initialized = TRUE;
+      g_variant_builder_add (&pack_builder, "u", GUINT32_TO_BE (0));
+      g_variant_builder_add (&pack_builder, "u", GUINT32_TO_BE (uid));
+      g_variant_builder_add (&pack_builder, "u", GUINT32_TO_BE (gid));
+      g_variant_builder_add (&pack_builder, "u", GUINT32_TO_BE (mode));
+
+      xattrs = ostree_get_xattrs_for_path (path, error);
+      if (!xattrs)
+        goto out;
+      g_variant_builder_add (&pack_builder, "@a(ayay)", xattrs);
+
+      if (S_ISREG (mode))
+        {
+          object_size = (guint64)g_file_info_get_size (finfo);
+        }
+      else if (S_ISLNK (mode))
+        {
+          target = g_file_info_get_attribute_byte_string (finfo, G_FILE_ATTRIBUTE_STANDARD_SYMLINK_TARGET);
+          object_size = strlen (target);
+        }
+      else if (S_ISBLK (mode) || S_ISCHR (mode))
+        {
+          device = g_file_info_get_attribute_uint32 (finfo, G_FILE_ATTRIBUTE_UNIX_DEVICE);
+          object_size = 4;
+        }
+      else
+        g_assert_not_reached ();
+
+      g_variant_builder_add (&pack_builder, "t", GUINT64_TO_BE (object_size));
+      pack_variant = g_variant_builder_end (&pack_builder);
+      pack_builder_initialized = FALSE;
+
+      metadata_size_be = GUINT32_TO_BE (g_variant_get_size (pack_variant));
+
+      if (!g_output_stream_write_all (output, &metadata_size_be, 4,
+                                      &bytes_written, cancellable, error))
+        goto out;
+      g_assert (bytes_written == 4);
+
+      if (!g_output_stream_write_all (output, g_variant_get_data (pack_variant), g_variant_get_size (pack_variant),
+                                      &bytes_written, cancellable, error))
+        goto out;
+
+      if (S_ISREG (mode))
+        {
+          instream = g_file_read (file, NULL, error);
+          if (!instream)
+            goto out;
+          bytes_written = g_output_stream_splice (output, (GInputStream*)instream, 0, cancellable, error);
+          if (bytes_written < 0)
+            goto out;
+          if (bytes_written != object_size)
+            {
+              g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                           "File size changed unexpectedly");
+              goto out;
+            }
+        }
+      else if (S_ISLNK (mode))
+        {
+          if (!g_output_stream_write_all (output, target, object_size,
+                                          &bytes_written, cancellable, error))
+            goto out;
+        }
+      else if (S_ISBLK (mode) || S_ISCHR (mode))
+        {
+          guint32 device_be = GUINT32_TO_BE (device);
+          g_assert (object_size == 4);
+          if (!g_output_stream_write_all (output, &device_be, object_size,
+                                          &bytes_written, cancellable, error))
+            goto out;
+          g_assert (bytes_written == 4);
+        }
+      else
+        g_assert_not_reached ();
+    }
+  
+  ret = TRUE;
+ out:
+  g_free (path);
+  g_clear_object (&finfo);
+  g_clear_object (&instream);
+  if (xattrs)
+    g_variant_unref (xattrs);
+  if (pack_builder_initialized)
+    g_variant_builder_clear (&pack_builder);
+  if (pack_variant)
+    g_variant_unref (pack_variant);
+  return ret;
+}
+

src/libostree/ostree-core.h

@@ -39,6 +39,7 @@ typedef enum {
   OSTREE_SERIALIZED_DIRMETA_VARIANT = 3,
   OSTREE_SERIALIZED_XATTR_VARIANT = 4
 } OstreeSerializedVariantType;
+#define OSTREE_SERIALIZED_VARIANT_LAST 4
 
 #define OSTREE_SERIALIZED_VARIANT_FORMAT "(uv)"
 
@@ -83,13 +84,54 @@ typedef enum {
  */
 #define OSTREE_COMMIT_GVARIANT_FORMAT "(ua{sv}ssstss)"
 
-GVariant *ostree_get_xattrs_for_path (const char *path,
-                                        GError    **error);
+gboolean ostree_validate_checksum_string (const char *sha256,
+                                          GError    **error);
+
+char *ostree_get_relative_object_path (const char *checksum,
+                                       OstreeObjectType type,
+                                       gboolean         archive);
+
+GVariant *ostree_get_xattrs_for_path (const char   *path,
+                                      GError     **error);
+
+gboolean ostree_set_xattrs (const char *path, GVariant *xattrs, GError **error);
+
+gboolean ostree_parse_metadata_file (const char                  *path,
+                                     OstreeSerializedVariantType *out_type,
+                                     GVariant                   **out_variant,
+                                     GError                     **error);
 
 gboolean ostree_stat_and_checksum_file (int dirfd, const char *path,
-                                          GChecksum **out_checksum,
-                                          struct stat *out_stbuf,
-                                          GError **error);
+                                        OstreeObjectType type,
+                                        GChecksum **out_checksum,
+                                        struct stat *out_stbuf,
+                                        GError **error);
+
+/* Packed files:
+ *
+ * guint32 metadata_length [metadata gvariant] [content]
+ *
+ * metadata variant:
+ * u - Version
+ * u - uid
+ * u - gid
+ * u - mode
+ * a(ayay) - xattrs
+ * t - content length
+ *
+ * And then following the end of the variant is the content.  If
+ * symlink, then this is the target; if device, then device ID as
+ * network byte order uint32.
+ */
+#define OSTREE_PACK_FILE_VARIANT_FORMAT "(uuuua(ayay)t)"
+
+gboolean  ostree_pack_object (GOutputStream     *output,
+                              GFile             *path,
+                              OstreeObjectType  objtype,
+                              GCancellable     *cancellable,
+                              GError          **error);
+
+void ostree_checksum_update_stat (GChecksum *checksum, guint32 uid, guint32 gid, guint32 mode);
 
 
 #endif /* _OSTREE_REPO */

src/libostree/ostree-repo.c

@@ -57,12 +57,14 @@ struct _OstreeRepoPrivate {
   char *path;
   GFile *repo_file;
   GFile *local_heads_dir;
+  GFile *remote_heads_dir;
   char *objects_path;
   char *config_path;
 
   gboolean inited;
 
   GKeyFile *config;
+  gboolean archive;
 };
 
 static void
@@ -74,6 +76,7 @@ ostree_repo_finalize (GObject *object)
   g_free (priv->path);
   g_clear_object (&priv->repo_file);
   g_clear_object (&priv->local_heads_dir);
+  g_clear_object (&priv->remote_heads_dir);
   g_free (priv->objects_path);
   g_free (priv->config_path);
   if (priv->config)
@@ -140,6 +143,7 @@ ostree_repo_constructor (GType                  gtype,
   
   priv->repo_file = ot_util_new_file_for_path (priv->path);
   priv->local_heads_dir = g_file_resolve_relative_path (priv->repo_file, "refs/heads");
+  priv->remote_heads_dir = g_file_resolve_relative_path (priv->repo_file, "refs/remotes");
   
   priv->objects_path = g_build_filename (priv->path, "objects", NULL);
   priv->config_path = g_build_filename (priv->path, "config", NULL);
@@ -179,19 +183,6 @@ ostree_repo_new (const char *path)
   return g_object_new (OSTREE_TYPE_REPO, "path", path, NULL);
 }
 
-static gboolean
-validate_checksum_string (const char *sha256,
-                          GError    **error)
-{
-  if (strlen (sha256) != 64)
-    {
-      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                   "Invalid rev '%s'", sha256);
-      return FALSE;
-    }
-  return TRUE;
-}
-
 static gboolean
 parse_rev_file (OstreeRepo     *self,
                 const char     *path,
@@ -252,7 +243,7 @@ parse_rev_file (OstreeRepo     *self,
     }
   else 
     {
-      if (!validate_checksum_string (rev, error))
+      if (!ostree_validate_checksum_string (rev, error))
         goto out;
     }
 
@@ -304,7 +295,7 @@ resolve_rev (OstreeRepo     *self,
        {
          g_strchomp (ret_rev);
          
-         if (!validate_checksum_string (ret_rev, error))
+         if (!ostree_validate_checksum_string (ret_rev, error))
            goto out;
        }
    }
@@ -357,12 +348,88 @@ write_checksum_file (GFile *parentdir,
   return ret;
 }
 
+/**
+ * ostree_repo_get_config:
+ * @self:
+ *
+ * Returns: (transfer none): The repository configuration; do not modify
+ */
+GKeyFile *
+ostree_repo_get_config (OstreeRepo *self)
+{
+  OstreeRepoPrivate *priv = GET_PRIVATE (self);
+
+  g_return_val_if_fail (priv->inited, NULL);
+
+  return priv->config;
+}
+
+/**
+ * ostree_repo_copy_config:
+ * @self:
+ *
+ * Returns: (transfer full): A newly-allocated copy of the repository config
+ */
+GKeyFile *
+ostree_repo_copy_config (OstreeRepo *self)
+{
+  OstreeRepoPrivate *priv = GET_PRIVATE (self);
+  GKeyFile *copy;
+  char *data;
+  gsize len;
+
+  g_return_val_if_fail (priv->inited, NULL);
+
+  copy = g_key_file_new ();
+  data = g_key_file_to_data (priv->config, &len, NULL);
+  if (!g_key_file_load_from_data (copy, data, len, 0, NULL))
+    g_assert_not_reached ();
+  g_free (data);
+  return copy;
+}
+
+/**
+ * ostree_repo_write_config:
+ * @self:
+ * @new_config: Overwrite the config file with this data.  Do not change later!
+ * @error: a #GError
+ *
+ * Save @new_config in place of this repository's config file.  Note
+ * that @new_config should not be modified after - this function
+ * simply adds a reference.
+ */
+gboolean
+ostree_repo_write_config (OstreeRepo *self,
+                          GKeyFile   *new_config,
+                          GError    **error)
+{
+  OstreeRepoPrivate *priv = GET_PRIVATE (self);
+  char *data = NULL;
+  gsize len;
+  gboolean ret = FALSE;
+
+  g_return_val_if_fail (priv->inited, FALSE);
+
+  data = g_key_file_to_data (new_config, &len, error);
+  if (!g_file_set_contents (priv->config_path, data, len, error))
+    goto out;
+  
+  g_key_file_unref (priv->config);
+  priv->config = g_key_file_ref (new_config);
+
+  ret = TRUE;
+ out:
+  g_free (data);
+  return ret;
+}
+
 gboolean
 ostree_repo_check (OstreeRepo *self, GError **error)
 {
   OstreeRepoPrivate *priv = GET_PRIVATE (self);
   gboolean ret = FALSE;
   char *version = NULL;;
+  GError *temp_error = NULL;
 
   g_return_val_if_fail (error == NULL || *error == NULL, FALSE);
 
@@ -383,9 +450,12 @@ ostree_repo_check (OstreeRepo *self, GError **error)
       goto out;
     }
 
-  version = g_key_file_get_value (priv->config, "core", "repo_version", error);
-  if (!version)
-    goto out;
+  version = g_key_file_get_value (priv->config, "core", "repo_version", &temp_error);
+  if (temp_error)
+    {
+      g_propagate_error (error, temp_error);
+      goto out;
+    }
 
   if (strcmp (version, "0") != 0)
     {
@@ -394,6 +464,20 @@ ostree_repo_check (OstreeRepo *self, GError **error)
       goto out;
     }
 
+  priv->archive = g_key_file_get_boolean (priv->config, "core", "archive", &temp_error);
+  if (temp_error)
+    {
+      if (g_error_matches (temp_error, G_KEY_FILE_ERROR, G_KEY_FILE_ERROR_NOT_FOUND))
+        {
+          g_clear_error (&temp_error);
+        }
+      else
+        {
+          g_propagate_error (error, temp_error);
+          goto out;
+        }
+    }
+
   priv->inited = TRUE;
   
   ret = TRUE;
@@ -402,6 +486,23 @@ ostree_repo_check (OstreeRepo *self, GError **error)
   return ret;
 }
 
+const char *
+ostree_repo_get_path (OstreeRepo  *self)
+{
+  OstreeRepoPrivate *priv = GET_PRIVATE (self);
+  return priv->path;
+}
+
+gboolean      
+ostree_repo_is_archive (OstreeRepo  *self)
+{
+  OstreeRepoPrivate *priv = GET_PRIVATE (self);
+
+  g_return_val_if_fail (priv->inited, FALSE);
+
+  return priv->archive;
+}
+
 static gboolean
 import_gvariant_object (OstreeRepo  *self,
                         OstreeSerializedVariantType type,
@@ -463,54 +564,13 @@ load_gvariant_object_unknown (OstreeRepo  *self,
                               GVariant     **out_variant,
                               GError       **error)
 {
-  GMappedFile *mfile = NULL;
   gboolean ret = FALSE;
-  GVariant *ret_variant = NULL;
-  GVariant *container = NULL;
   char *path = NULL;
-  guint32 ret_type;
 
   path = get_object_path (self, sha256, OSTREE_OBJECT_TYPE_META);
-  
-  mfile = g_mapped_file_new (path, FALSE, error);
-  if (mfile == NULL)
-    goto out;
-  else
-    {
-      container = g_variant_new_from_data (G_VARIANT_TYPE (OSTREE_SERIALIZED_VARIANT_FORMAT),
-                                           g_mapped_file_get_contents (mfile),
-                                           g_mapped_file_get_length (mfile),
-                                           FALSE,
-                                           (GDestroyNotify) g_mapped_file_unref,
-                                           mfile);
-      if (!g_variant_is_of_type (container, G_VARIANT_TYPE (OSTREE_SERIALIZED_VARIANT_FORMAT)))
-        {
-          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                       "Corrupted metadata object '%s'", sha256);
-          goto out;
-        }
-      g_variant_get (container, "(uv)",
-                     &ret_type, &ret_variant);
-      mfile = NULL;
-    }
-
-  ret = TRUE;
- out:
-  if (!ret)
-    {
-      if (ret_variant)
-        g_variant_unref (ret_variant);
-    }
-  else
-    {
-      *out_type = ret_type;
-      *out_variant = ret_variant;
-    }
-  if (container != NULL)
-    g_variant_unref (container);
+  ret = ostree_parse_metadata_file (path, out_type, out_variant, error);
   g_free (path);
-  if (mfile != NULL)
-    g_mapped_file_unref (mfile);
+
   return ret;
 }
 
@@ -534,7 +594,6 @@ load_gvariant_object (OstreeRepo  *self,
                    "Corrupted metadata object '%s'; found type %u, expected %u", sha256,
                    type, (guint32)expected_type);
       goto out;
-      
     }
 
   ret = TRUE;
@@ -616,41 +675,26 @@ get_object_path (OstreeRepo  *self,
                  OstreeObjectType type)
 {
   OstreeRepoPrivate *priv = GET_PRIVATE (self);
-  char *checksum_prefix;
-  char *base_path;
   char *ret;
-  const char *type_string;
+  char *relpath;
 
-  checksum_prefix = g_strndup (checksum, 2);
-  base_path = g_build_filename (priv->objects_path, checksum_prefix, checksum + 2, NULL);
-  switch (type)
-    {
-    case OSTREE_OBJECT_TYPE_FILE:
-      type_string = ".file";
-      break;
-    case OSTREE_OBJECT_TYPE_META:
-      type_string = ".meta";
-      break;
-    default:
-      g_assert_not_reached ();
-    }
-  ret = g_strconcat (base_path, type_string, NULL);
-  g_free (base_path);
-  g_free (checksum_prefix);
+  relpath = ostree_get_relative_object_path (checksum, type, priv->archive);
+  ret = g_build_filename (priv->path, relpath, NULL);
+  g_free (relpath);
  
   return ret;
 }
 
 static char *
 prepare_dir_for_checksum_get_object_path (OstreeRepo *self,
-                                          GChecksum    *checksum,
+                                          const char   *checksum,
                                           OstreeObjectType type,
                                           GError      **error)
 {
   char *checksum_dir = NULL;
   char *object_path = NULL;
 
-  object_path = get_object_path (self, g_checksum_get_string (checksum), type);
+  object_path = get_object_path (self, checksum, type);
   checksum_dir = g_path_get_dirname (object_path);
 
   if (!ot_util_ensure_directory (checksum_dir, FALSE, error))
@@ -662,21 +706,23 @@ prepare_dir_for_checksum_get_object_path (OstreeRepo *self,
 }
 
 static gboolean
-link_one_file (OstreeRepo *self, const char *path, OstreeObjectType type,
-               gboolean ignore_exists, gboolean force,
-               GChecksum **out_checksum,
-               GError **error)
+link_object_trusted (OstreeRepo   *self,
+                     const char   *path,
+                     const char   *checksum,
+                     OstreeObjectType objtype,
+                     gboolean      ignore_exists,
+                     gboolean      force,
+                     gboolean     *did_exist,
+                     GError      **error)
 {
   char *src_basename = NULL;
   char *src_dirname = NULL;
   char *dest_basename = NULL;
   char *tmp_dest_basename = NULL;
   char *dest_dirname = NULL;
-  GChecksum *id = NULL;
   DIR *src_dir = NULL;
   DIR *dest_dir = NULL;
   gboolean ret = FALSE;
-  struct stat stbuf;
   char *dest_path = NULL;
 
   src_basename = g_path_get_basename (path);
@@ -689,9 +735,7 @@ link_one_file (OstreeRepo *self, const char *path, OstreeObjectType type,
       goto out;
     }
 
-  if (!ostree_stat_and_checksum_file (dirfd (src_dir), path, &id, &stbuf, error))
-    goto out;
-  dest_path = prepare_dir_for_checksum_get_object_path (self, id, type, error);
+  dest_path = prepare_dir_for_checksum_get_object_path (self, checksum, objtype, error);
   if (!dest_path)
     goto out;
 
@@ -719,7 +763,11 @@ link_one_file (OstreeRepo *self, const char *path, OstreeObjectType type,
           ot_util_set_error_from_errno (error, errno);
           goto out;
         }
+      else
+        *did_exist = TRUE;
     }
+  else
+    *did_exist = FALSE;
 
   if (force)
     {
@@ -732,12 +780,8 @@ link_one_file (OstreeRepo *self, const char *path, OstreeObjectType type,
       (void) unlinkat (dirfd (dest_dir), tmp_dest_basename, 0);
     }
 
-  *out_checksum = id;
-  id = NULL;
   ret = TRUE;
  out:
-  if (id != NULL)
-    g_checksum_free (id);
   if (src_dir != NULL)
     closedir (src_dir);
   if (dest_dir != NULL)
@@ -750,12 +794,108 @@ link_one_file (OstreeRepo *self, const char *path, OstreeObjectType type,
   return ret;
 }
 
+static gboolean
+archive_file_trusted (OstreeRepo   *self,
+                      const char   *path,
+                      const char   *checksum,
+                      OstreeObjectType objtype,
+                      gboolean      ignore_exists,
+                      gboolean      force,
+                      gboolean     *did_exist,
+                      GError      **error)
+{
+  GFile *infile = NULL;
+  GFile *outfile = NULL;
+  GFileOutputStream *out = NULL;
+  gboolean ret = FALSE;
+  char *dest_path = NULL;
+  char *dest_tmp_path = NULL;
+
+  infile = ot_util_new_file_for_path (path);
+
+  dest_path = prepare_dir_for_checksum_get_object_path (self, checksum, objtype, error);
+  if (!dest_path)
+    goto out;
+
+  dest_tmp_path = g_strconcat (dest_path, ".tmp", NULL);
+
+  outfile = ot_util_new_file_for_path (dest_tmp_path);
+  out = g_file_replace (outfile, NULL, FALSE, 0, NULL, error);
+  if (!out)
+    goto out;
+
+  if (!ostree_pack_object ((GOutputStream*)out, infile, objtype, NULL, error))
+    goto out;
+  
+  if (!g_output_stream_close ((GOutputStream*)out, NULL, error))
+    goto out;
+
+  if (rename (dest_tmp_path, dest_path) < 0)
+    {
+      ot_util_set_error_from_errno (error, errno);
+      goto out;
+    }
+
+  ret = TRUE;
+ out:
+  g_free (dest_path);
+  g_free (dest_tmp_path);
+  g_clear_object (&infile);
+  g_clear_object (&outfile);
+  g_clear_object (&out);
+  return ret;
+}
+  
+gboolean      
+ostree_repo_store_object_trusted (OstreeRepo   *self,
+                                  const char   *path,
+                                  const char   *checksum,
+                                  OstreeObjectType objtype,
+                                  gboolean      ignore_exists,
+                                  gboolean      force,
+                                  gboolean     *did_exist,
+                                  GError      **error)
+{
+  OstreeRepoPrivate *priv = GET_PRIVATE (self);
+  if (priv->archive && objtype == OSTREE_OBJECT_TYPE_FILE)
+    return archive_file_trusted (self, path, checksum, objtype, ignore_exists, force, did_exist, error);
+  else
+    return link_object_trusted (self, path, checksum, objtype, ignore_exists, force, did_exist, error);
+}
+
+static gboolean
+link_one_file (OstreeRepo *self, const char *path, OstreeObjectType type,
+               gboolean ignore_exists, gboolean force,
+               GChecksum **out_checksum,
+               GError **error)
+{
+  gboolean ret = FALSE;
+  struct stat stbuf;
+  GChecksum *id = NULL;
+  gboolean did_exist;
+
+  if (!ostree_stat_and_checksum_file (-1, path, type, &id, &stbuf, error))
+    goto out;
+
+  if (!ostree_repo_store_object_trusted (self, path, g_checksum_get_string (id), type,
+                                         ignore_exists, force, &did_exist, error))
+    goto out;
+
+  *out_checksum = id;
+  id = NULL;
+  ret = TRUE;
+ out:
+  if (id != NULL)
+    g_checksum_free (id);
+  return ret;
+}
+
 gboolean
 ostree_repo_link_file (OstreeRepo *self,
-                         const char   *path,
-                         gboolean      ignore_exists,
-                         gboolean      force,
-                         GError      **error)
+                       const char   *path,
+                       gboolean      ignore_exists,
+                       gboolean      force,
+                       GError      **error)
 {
   OstreeRepoPrivate *priv = GET_PRIVATE (self);
   GChecksum *checksum = NULL;
@@ -770,6 +910,227 @@ ostree_repo_link_file (OstreeRepo *self,
   return TRUE;
 }
 
+static gboolean
+unpack_and_checksum_packfile (OstreeRepo   *self,
+                              const char   *path,
+                              gchar       **out_filename,
+                              GChecksum   **out_checksum,
+                              GError      **error)
+{
+  OstreeRepoPrivate *priv = GET_PRIVATE (self);
+  gboolean ret = FALSE;
+  GFile *file = NULL;
+  char *temp_path = NULL;
+  GFile *temp_file = NULL;
+  GFileOutputStream *temp_out = NULL;
+  char *metadata_buf = NULL;
+  GVariant *metadata = NULL;
+  GVariant *xattrs = NULL;
+  GFileInputStream *in = NULL;
+  GChecksum *ret_checksum = NULL;
+  guint32 metadata_len;
+  guint32 version, uid, gid, mode;
+  guint64 content_len;
+  gsize bytes_read, bytes_written;
+  char buf[8192];
+  int temp_fd = -1;
+
+  file = ot_util_new_file_for_path (path);
+
+  in = g_file_read (file, NULL, error);
+  if (!in)
+    goto out;
+      
+  if (!g_input_stream_read_all ((GInputStream*)in, &metadata_len, 4, &bytes_read, NULL, error))
+    goto out;
+  if (bytes_read != 4)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Corrupted packfile; too short while reading metadata length");
+      goto out;
+    }
+      
+  metadata_len = GUINT32_FROM_BE (metadata_len);
+  metadata_buf = g_malloc (metadata_len);
+
+  if (!g_input_stream_read_all ((GInputStream*)in, metadata_buf, metadata_len, &bytes_read, NULL, error))
+    goto out;
+  if (bytes_read != metadata_len)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Corrupted packfile; too short while reading metadata");
+      goto out;
+    }
+
+  metadata = g_variant_new_from_data (G_VARIANT_TYPE (OSTREE_PACK_FILE_VARIANT_FORMAT),
+                                      metadata_buf, metadata_len, FALSE, NULL, NULL);
+      
+  g_variant_get (metadata, "(uuuu@a(ayay)t)",
+                 &version, &uid, &gid, &mode,
+                 &xattrs, &content_len);
+  uid = GUINT32_FROM_BE (uid);
+  gid = GUINT32_FROM_BE (gid);
+  mode = GUINT32_FROM_BE (mode);
+  content_len = GUINT64_FROM_BE (content_len);
+
+  temp_path = g_build_filename (priv->path, "tmp-packfile-XXXXXX");
+  temp_file = ot_util_new_file_for_path (temp_path);
+      
+  ret_checksum = g_checksum_new (G_CHECKSUM_SHA256);
+
+  if (S_ISREG (mode))
+    {
+      temp_fd = g_mkstemp (temp_path);
+      if (temp_fd < 0)
+        {
+          ot_util_set_error_from_errno (error, errno);
+          goto out;
+        }
+      close (temp_fd);
+      temp_fd = -1;
+      temp_out = g_file_replace (temp_file, NULL, FALSE, 0, NULL, error);
+      if (!temp_out)
+        goto out;
+
+      do
+        {
+          if (!g_input_stream_read_all ((GInputStream*)in, buf, sizeof(buf), &bytes_read, NULL, error))
+            goto out;
+          g_checksum_update (ret_checksum, (guint8*)buf, bytes_read);
+          if (!g_output_stream_write_all ((GOutputStream*)temp_out, buf, bytes_read, &bytes_written, NULL, error))
+            goto out;
+        }
+      while (bytes_read > 0);
+
+      if (!g_output_stream_close ((GOutputStream*)temp_out, NULL, error))
+        goto out;
+    }
+  else if (S_ISLNK (mode))
+    {
+      g_assert (sizeof (buf) > PATH_MAX);
+
+      if (!g_input_stream_read_all ((GInputStream*)in, buf, sizeof(buf), &bytes_read, NULL, error))
+        goto out;
+      buf[bytes_read] = '\0';
+      if (symlink (buf, temp_path) < 0)
+        {
+          ot_util_set_error_from_errno (error, errno);
+          goto out;
+        }
+    }
+  else if (S_ISCHR (mode) || S_ISBLK (mode))
+    {
+      guint32 dev;
+
+      if (!g_input_stream_read_all ((GInputStream*)in, &dev, 4, &bytes_read, NULL, error))
+        goto out;
+      if (bytes_read != 4)
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "Corrupted packfile; too short while reading device id");
+          goto out;
+        }
+      dev = GUINT32_FROM_BE (dev);
+      if (mknod (temp_path, mode, dev) < 0)
+        {
+          ot_util_set_error_from_errno (error, errno);
+          goto out;
+        }
+    }
+  else
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Corrupted packfile; invalid mode %u", mode);
+      goto out;
+    }
+
+  if (!S_ISLNK (mode))
+    {
+      if (chmod (temp_path, mode) < 0)
+        {
+          ot_util_set_error_from_errno (error, errno);
+          goto out;
+        }
+    }
+
+  if (!ostree_set_xattrs (temp_path, xattrs, error))
+    goto out;
+
+  ostree_checksum_update_stat (ret_checksum, uid, gid, mode);
+  g_checksum_update (ret_checksum, (guint8*)g_variant_get_data (xattrs), g_variant_get_size (xattrs));
+
+  ret = TRUE;
+  *out_checksum = ret_checksum;
+  ret_checksum = NULL;
+  *out_filename = temp_path;
+  temp_path = NULL;
+ out:
+  if (ret_checksum)
+    g_checksum_free (ret_checksum);
+  g_free (metadata_buf);
+  if (temp_path)
+    (void) unlink (temp_path);
+  g_free (temp_path);
+  g_clear_object (&file);
+  g_clear_object (&in);
+  g_clear_object (&temp_file);
+  g_clear_object (&temp_out);
+  if (metadata)
+   g_variant_unref (metadata);
+  if (xattrs)
+    g_variant_unref (xattrs);
+  return ret;
+}
+
+gboolean
+ostree_repo_store_packfile (OstreeRepo       *self,
+                            const char       *expected_checksum,
+                            const char       *path,
+                            OstreeObjectType  objtype,
+                            GError          **error)
+{
+  gboolean ret = FALSE;
+  char *tempfile = NULL;
+  GChecksum *checksum = NULL;
+  struct stat stbuf;
+  gboolean did_exist;
+
+  if (objtype == OSTREE_OBJECT_TYPE_META)
+    {
+      if (!ostree_stat_and_checksum_file (-1, path, objtype, &checksum, &stbuf, error))
+        goto out;
+    }
+  else
+    {
+      if (!unpack_and_checksum_packfile (self, path, &tempfile, &checksum, error))
+        goto out;
+
+    }
+
+  if (strcmp (g_checksum_get_string (checksum), expected_checksum) != 0)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Corrupted object %s (actual checksum is %s)",
+                   expected_checksum, g_checksum_get_string (checksum));
+      goto out;
+    }
+
+  if (!ostree_repo_store_object_trusted (self, tempfile ? tempfile : path,
+                                         expected_checksum,
+                                         objtype,
+                                         TRUE, FALSE, &did_exist, error))
+    goto out;
+
+  ret = TRUE;
+ out:
+  if (tempfile)
+    (void) unlink (tempfile);
+  g_free (tempfile);
+  if (checksum)
+    g_checksum_free (checksum);
+  return ret;
+}
+
 typedef struct _ParsedTreeData ParsedTreeData;
 typedef struct _ParsedDirectoryData ParsedDirectoryData;
 
@@ -839,6 +1200,7 @@ parse_tree (OstreeRepo    *self,
                              sha256, &tree_variant, error))
     goto out;
 
+  /* PARSE OSTREE_SERIALIZED_TREE_VARIANT */
   g_variant_get (tree_variant, "(u@a{sv}@a(ss)@a(sss))",
                  &version, &meta_variant, &files_variant, &dirs_variant);
 
@@ -922,6 +1284,7 @@ load_commit_and_trees (OstreeRepo   *self,
                              commit_sha256, &ret_commit, error))
     goto out;
 
+  /* PARSE OSTREE_SERIALIZED_COMMIT_VARIANT */
   g_variant_get_child (ret_commit, 6, "&s", &tree_contents_checksum);
   g_variant_get_child (ret_commit, 7, "&s", &tree_meta_checksum);
 
@@ -1366,6 +1729,18 @@ add_files_to_tree_and_import (OstreeRepo   *self,
   return ret;
 }
 
+gboolean      
+ostree_repo_write_ref (OstreeRepo  *self,
+                       gboolean     is_local,
+                       const char  *name,
+                       const char  *rev,
+                       GError     **error)
+{
+  OstreeRepoPrivate *priv = GET_PRIVATE (self);
+  return write_checksum_file (is_local ? priv->local_heads_dir : priv->remote_heads_dir, 
+                              name, rev, error);
+}
+
 static gboolean
 commit_parsed_tree (OstreeRepo *self,
                     const char   *branch,
@@ -1377,7 +1752,6 @@ commit_parsed_tree (OstreeRepo *self,
                     GChecksum   **out_commit,
                     GError      **error)
 {
-  OstreeRepoPrivate *priv = GET_PRIVATE (self);
   gboolean ret = FALSE;
   GChecksum *root_checksum = NULL;
   GChecksum *ret_commit = NULL;
@@ -1403,7 +1777,7 @@ commit_parsed_tree (OstreeRepo *self,
                                commit, &ret_commit, error))
     goto out;
 
-  if (!write_checksum_file (priv->local_heads_dir, branch, g_checksum_get_string (ret_commit), error))
+  if (!ostree_repo_write_ref (self, TRUE, branch, g_checksum_get_string (ret_commit), error))
     goto out;
 
   ret = TRUE;
@@ -1648,7 +2022,8 @@ iter_object_dir (OstreeRepo   *self,
       
       if (type != G_FILE_TYPE_DIRECTORY
           && (g_str_has_suffix (name, ".meta")
-              || g_str_has_suffix (name, ".file")))
+              || g_str_has_suffix (name, ".file")
+              || g_str_has_suffix (name, ".packfile")))
         {
           char *dot;
           char *path;
@@ -1789,6 +2164,7 @@ checkout_one_directory (OstreeRepo  *self,
 
   dest_path = g_build_filename (destination, dirname, NULL);
       
+  /* PARSE OSTREE_SERIALIZED_DIRMETA_VARIANT */
   g_variant_get (dir->meta_data, "(uuuu@a(ayay))",
                  &version, &uid, &gid, &mode,
                  &xattr_variant);

src/libostree/ostree-repo.h

@@ -53,17 +53,50 @@ OstreeRepo* ostree_repo_new (const char *path);
 
 gboolean      ostree_repo_check (OstreeRepo  *self, GError **error);
 
+const char *  ostree_repo_get_path (OstreeRepo  *self);
+
+gboolean      ostree_repo_is_archive (OstreeRepo  *self);
+
+GKeyFile *    ostree_repo_get_config (OstreeRepo *self);
+
+GKeyFile *    ostree_repo_copy_config (OstreeRepo *self);
+
+gboolean      ostree_repo_write_config (OstreeRepo *self,
+                                        GKeyFile   *new_config,
+                                        GError    **error);
+
 gboolean      ostree_repo_link_file (OstreeRepo *self,
-                                       const char   *path,
-                                       gboolean      ignore_exists,
-                                       gboolean      force,
-                                       GError      **error);
+                                     const char   *path,
+                                     gboolean      ignore_exists,
+                                     gboolean      force,
+                                     GError      **error);
+
+gboolean      ostree_repo_store_packfile (OstreeRepo       *self,
+                                           const char       *expected_checksum,
+                                           const char       *path,
+                                           OstreeObjectType  objtype,
+                                           GError          **error);
+
+gboolean      ostree_repo_store_object_trusted (OstreeRepo   *self,
+                                                const char   *path,
+                                                const char   *checksum,
+                                                OstreeObjectType objtype,
+                                                gboolean      ignore_exists,
+                                                gboolean      force,
+                                                gboolean     *did_exist,
+                                                GError      **error);
 
 gboolean      ostree_repo_resolve_rev (OstreeRepo  *self,
                                        const char  *rev,
                                        char       **out_resolved,
                                        GError     **error);
 
+gboolean      ostree_repo_write_ref (OstreeRepo  *self,
+                                     gboolean     is_local,
+                                     const char  *name,
+                                     const char  *rev,
+                                     GError     **error);
+
 gboolean      ostree_repo_load_variant (OstreeRepo *self,
                                           const char   *sha256,
                                           OstreeSerializedVariantType *out_type,