core: Add pack files

This concept is also directly inspired by git.  At present, our
implementation is quite similar, except we don't have delta
compression.

src/libostree/README.md

@@ -0,0 +1,78 @@
+Repository design
+-----------------
+
+At the heart of OSTree is the repository.  It's very similar to git,
+with the idea of content-addressed storage.  However, OSTree is
+designed to store operating system binaries, not source code.  There
+are several consequences to this.  The key difference as compared to
+git is that the OSTree definition of "content" includes key Unix
+metadata such as owner uid/gid, as well as all extended attributes.
+
+Essentially OSTree is designed so that if two files have the same
+OSTree checksum, it's safe to replace them with a hard link.  This
+fundamental design means that an OSTree repository imposes negligible
+overhead.  In contrast, a git repository stores copies of
+zlib-compressed data.
+
+Key differences versus git
+--------------------------
+
+ * As mentioned above, extended attributes and owner uid/gid are versioned
+ * Optimized for Unix hardlinks between repository and checkout
+ * SHA256 instead of SHA1
+ * Support for empty directories
+
+Binary files
+------------
+
+While this is still in planning, I plan to heavily optimize OSTree for
+versioning ELF operating systems.  In industry jargon, this would be
+"content-aware storage".
+
+Trimming history
+----------------
+
+OSTree will also be optimized to trim intermediate history; in theory
+one can regenerate binaries from corresponding (git) source code, so
+we don't need to keep all possible builds over time.
+
+MILESTONE 1
+-----------
+* Basic pack files (like git)
+
+MILESTONE 2
+-----------
+* Store checksums as ay
+* Drop version/metadata from tree/dirmeta objects
+* Split pack files into metadata/data
+* Restructure repository so that links can be generated as a cache;
+  i.e. objects/raw, pack files are now the canonical
+* For files, checksum combination of metadata variant + raw data 
+  - i.e. there is only OSTREE_OBJECT_TYPE_FILE (again)
+
+MILESTONE 3
+-----------
+
+* Drop archive/raw distinction - archive repositories always generate
+  packfiles per commit
+* Include git packv4 ideas:
+  - metadata packfiles have string dictionary (tree filenames and checksums)
+  - data packfiles match up similar objects
+* Rolling checksums for partitioning large files?  Kernel debuginfo
+* Improved pack clustering
+  - file fingerprinting?
+* ELF-x86 aware deltas
+
+Related work in storage
+-----------------------
+
+git: http://git-scm.com/
+Venti: http://plan9.bell-labs.com/magic/man2html/6/venti
+Elephant FS: http://www.hpl.hp.com/personal/Alistair_Veitch/papers/elephant-hotos/index.html
+
+Compression
+-----------
+
+xdelta: http://xdelta.org/
+Bsdiff: http://www.daemonology.net/bsdiff/
+xz: http://tukaani.org/xz/

src/libostree/ostree-core.c

@@ -28,6 +28,9 @@
 #include <sys/types.h>
 #include <attr/xattr.h>
 
+#define ALIGN_VALUE(this, boundary) \
+  (( ((unsigned long)(this)) + (((unsigned long)(boundary)) -1)) & (~(((unsigned long)(boundary))-1)))
+
 gboolean
 ostree_validate_checksum_string (const char *sha256,
                                  GError    **error)
@@ -486,6 +489,34 @@ ostree_set_xattrs (GFile  *f,
   return ret;
 }
 
+gboolean
+ostree_unwrap_metadata (GVariant              *container,
+                        OstreeObjectType       expected_type,
+                        GVariant             **out_variant,
+                        GError               **error)
+{
+  gboolean ret = FALSE;
+  GVariant *ret_variant = NULL;
+  guint32 actual_type;
+
+  g_variant_get (container, "(uv)",
+                 &actual_type, &ret_variant);
+  actual_type = GUINT32_FROM_BE (actual_type);
+  if (actual_type != expected_type)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Corrupted metadata object; found type %u, expected %u",
+                   actual_type, (guint32)expected_type);
+      goto out;
+    }
+
+  ret = TRUE;
+  ot_transfer_out_value (out_variant, &ret_variant);
+ out:
+  ot_clear_gvariant (&ret_variant);
+  return ret;
+}
+
 gboolean
 ostree_map_metadata_file (GFile                       *file,
                           OstreeObjectType             expected_type,
@@ -495,22 +526,16 @@ ostree_map_metadata_file (GFile                       *file,
   gboolean ret = FALSE;
   GVariant *ret_variant = NULL;
   GVariant *container = NULL;
-  guint32 actual_type;
 
   if (!ot_util_variant_map (file, OSTREE_SERIALIZED_VARIANT_FORMAT,
                             &container, error))
     goto out;
 
-  g_variant_get (container, "(uv)",
-                 &actual_type, &ret_variant);
-  ot_util_variant_take_ref (ret_variant);
-  actual_type = GUINT32_FROM_BE (actual_type);
-  if (actual_type != expected_type)
+  if (!ostree_unwrap_metadata (container, expected_type, &ret_variant,
+                               error))
     {
-      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                   "Corrupted metadata object '%s'; found type %u, expected %u",
-                   ot_gfile_get_path_cached (file), 
-                   actual_type, (guint32)expected_type);
+      g_prefix_error (error, "While parsing '%s': ",
+                      ot_gfile_get_path_cached (file));
       goto out;
     }
 
@@ -584,6 +609,142 @@ ostree_object_from_string (const char *str,
   *out_objtype = ostree_object_type_from_string (dot + 1);
 }
 
+guint
+ostree_hash_object_name (gconstpointer a)
+{
+  GVariant *variant = (gpointer)a;
+  const char *checksum;
+  OstreeObjectType objtype;
+  gint objtype_int;
+  
+  ostree_object_name_deserialize (variant, &checksum, &objtype);
+  objtype_int = (gint) objtype;
+  return g_str_hash (checksum) + g_int_hash (&objtype_int);
+}
+
+int
+ostree_cmp_checksum_bytes (GVariant *a,
+                           GVariant *b)
+{
+  gconstpointer a_data;
+  gconstpointer b_data;
+  gsize a_n_elts;
+  gsize b_n_elts;
+  
+  a_data = g_variant_get_fixed_array (a, &a_n_elts, 1);
+  g_assert (a_n_elts == 32);
+  b_data = g_variant_get_fixed_array (b, &b_n_elts, 1);
+  g_assert (b_n_elts == 32);
+
+  return memcmp (a_data, b_data, 32);
+}
+
+
+GVariant *
+ostree_object_name_serialize (const char *checksum,
+                              OstreeObjectType objtype)
+{
+  return g_variant_new ("(su)", checksum, (guint32)objtype);
+}
+
+void
+ostree_object_name_deserialize (GVariant         *variant,
+                                const char      **out_checksum,
+                                OstreeObjectType *out_objtype)
+{
+  guint32 objtype_u32;
+  g_variant_get (variant, "(&su)", out_checksum, &objtype_u32);
+  *out_objtype = (OstreeObjectType)objtype_u32;
+}
+
+GVariant *
+ostree_checksum_to_bytes (const char *sha256)
+{
+  guchar result[32];
+  guint i;
+  guint j;
+
+  for (i = 0, j = 0; i < 32; i += 1, j += 2)
+    {
+      gint big, little;
+
+      g_assert (sha256[j]);
+      g_assert (sha256[j+1]);
+
+      big = g_ascii_xdigit_value (sha256[j]);
+      little = g_ascii_xdigit_value (sha256[j+1]);
+
+      g_assert (big != -1);
+      g_assert (little != -1);
+
+      result[i] = (big << 4) | little;
+    }
+  
+  return g_variant_new_fixed_array (G_VARIANT_TYPE ("y"),
+                                    (guchar*)result, 32, 1);
+}
+
+char *
+ostree_checksum_from_bytes (GVariant *csum_bytes)
+{
+  static const gchar hexchars[] = "0123456789abcdef";
+  char *ret;
+  const guchar *bytes;
+  gsize n_elts;
+  guint i, j;
+
+  bytes = g_variant_get_fixed_array (csum_bytes, &n_elts, 1);
+  g_assert (n_elts == 32);
+
+  ret = g_malloc (65);
+  
+  for (i = 0, j = 0; i < 32; i++, j += 2)
+    {
+      guchar byte = bytes[i];
+      ret[j] = hexchars[byte >> 4];
+      ret[j+1] = hexchars[byte & 0xF];
+    }
+  ret[j] = '\0';
+
+  return ret;
+}
+
+GVariant *
+ostree_object_name_serialize_v2 (const char        *checksum,
+                                 OstreeObjectType   objtype)
+{
+  return g_variant_new ("(u@ay)", (guint32)objtype, ostree_checksum_to_bytes (checksum));
+}
+
+void
+ostree_object_name_deserialize_v2_hex (GVariant         *variant,
+                                       char            **out_checksum,
+                                       OstreeObjectType *out_objtype)
+{
+  GVariant *csum_bytes;
+  guint32 objtype_u32;
+
+  g_variant_get (variant, "(u@ay)", &objtype_u32, &csum_bytes);
+  g_variant_ref_sink (csum_bytes);
+  *out_checksum = ostree_checksum_from_bytes (csum_bytes);
+  g_variant_unref (csum_bytes);
+  *out_objtype = (OstreeObjectType)objtype_u32;
+}
+
+void
+ostree_object_name_deserialize_v2_bytes (GVariant         *variant,
+                                         const guchar    **out_checksum,
+                                         OstreeObjectType *out_objtype)
+{
+  GVariant *csum_bytes;
+  guint32 objtype_u32;
+  gsize n_elts;
+
+  g_variant_get (variant, "(u@ay)", &objtype_u32, &csum_bytes);
+  *out_checksum = (guchar*)g_variant_get_fixed_array (csum_bytes, &n_elts, 1);
+  *out_objtype = (OstreeObjectType)objtype_u32;
+}
+
 char *
 ostree_get_relative_object_path (const char *checksum,
                                  OstreeObjectType type)
@@ -1074,3 +1235,350 @@ ostree_create_temp_hardlink (GFile            *dir,
   g_clear_object (&possible_file);
   return ret;
 }
+
+gboolean
+ostree_read_pack_entry_raw (guchar        *pack_data,
+                            guint64        pack_len,
+                            guint64        offset,
+                            gboolean       trusted,
+                            GVariant     **out_entry,
+                            GCancellable  *cancellable,
+                            GError       **error)
+{
+  gboolean ret = FALSE;
+  GVariant *ret_entry = NULL;
+  guint64 entry_start;
+  guint64 entry_end;
+  guint32 entry_len;
+
+  if (G_UNLIKELY (!(offset <= pack_len)))
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Corrupted pack index; out of range offset %" G_GUINT64_FORMAT,
+                   offset);
+      goto out;
+    }
+  if (G_UNLIKELY (!((offset & 0x3) == 0)))
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Corrupted pack index; unaligned offset %" G_GUINT64_FORMAT,
+                   offset);
+      goto out;
+    }
+
+  entry_start = ALIGN_VALUE (offset + 4, 8);
+  if (G_UNLIKELY (!(entry_start <= pack_len)))
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Corrupted pack index; out of range data offset %" G_GUINT64_FORMAT,
+                   entry_start);
+      goto out;
+    }
+
+  g_assert ((((guint64)pack_data+offset) & 0x3) == 0);
+  entry_len = GUINT32_FROM_BE (*((guint32*)(pack_data+offset)));
+
+  entry_end = entry_start + entry_len;
+  if (G_UNLIKELY (!(entry_end <= pack_len)))
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Corrupted pack index; out of range entry length %u",
+                   entry_len);
+      goto out;
+    }
+
+  ret_entry = g_variant_new_from_data (OSTREE_PACK_FILE_CONTENT_VARIANT_FORMAT,
+                                       pack_data+entry_start, entry_len,
+                                       trusted, NULL, NULL);
+  ret = TRUE;
+  ot_transfer_out_value (out_entry, &ret_entry);
+ out:
+  ot_clear_gvariant (&ret_entry);
+  return ret;
+}
+
+GInputStream *
+ostree_read_pack_entry_as_stream (GVariant *pack_entry)
+{
+  GInputStream *memory_input;
+  GInputStream *ret_input = NULL;
+  GVariant *pack_data = NULL;
+  guchar entry_flags;
+  gconstpointer data_ptr;
+  gsize data_len;
+
+  g_variant_get_child (pack_entry, 1, "y", &entry_flags);
+  g_variant_get_child (pack_entry, 3, "@ay", &pack_data);
+
+  data_ptr = g_variant_get_fixed_array (pack_data, &data_len, 1);
+  memory_input = g_memory_input_stream_new_from_data (data_ptr, data_len, NULL);
+  g_object_set_data_full ((GObject*)memory_input, "ostree-mem-gvariant",
+                          pack_data, (GDestroyNotify) g_variant_unref);
+
+  if (entry_flags & OSTREE_PACK_FILE_ENTRY_FLAG_GZIP)
+    {
+      GConverter *decompressor;
+
+      decompressor = (GConverter*)g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP);
+      ret_input = (GInputStream*)g_object_new (G_TYPE_CONVERTER_INPUT_STREAM,
+                                               "converter", decompressor,
+                                               "base-stream", memory_input,
+                                               "close-base-stream", TRUE,
+                                               NULL);
+      g_object_unref (decompressor);
+    }
+  else
+    {
+      ret_input = memory_input;
+      memory_input = NULL;
+    }
+
+  return ret_input;
+}
+
+gboolean
+ostree_read_pack_entry_variant (GVariant            *pack_entry,
+                                OstreeObjectType     expected_objtype,
+                                gboolean             trusted,
+                                GVariant           **out_variant,
+                                GCancellable        *cancellable,
+                                GError             **error)
+{
+  gboolean ret = FALSE;
+  GInputStream *stream = NULL;
+  GVariant *container_variant = NULL;
+  GVariant *ret_variant = NULL;
+  guint32 actual_type;
+
+  stream = ostree_read_pack_entry_as_stream (pack_entry);
+  
+  if (!ot_util_variant_from_stream (stream, OSTREE_SERIALIZED_VARIANT_FORMAT,
+                                    trusted, &container_variant, cancellable, error))
+    goto out;
+
+  g_variant_ref_sink (container_variant);
+
+  g_variant_get (container_variant, "(uv)",
+                 &actual_type, &ret_variant);
+  actual_type = GUINT32_FROM_BE (actual_type);
+
+  if (actual_type != expected_objtype)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Corrupted metadata object in pack file; found type %u, expected %u",
+                   actual_type, (guint32)expected_objtype);
+      goto out;
+    }
+
+  ret = TRUE;
+  ot_transfer_out_value (out_variant, &ret_variant);
+ out:
+  g_clear_object (&stream);
+  ot_clear_gvariant (&ret_variant);
+  ot_clear_gvariant (&container_variant);
+  return ret;
+}
+
+gboolean
+ostree_pack_index_search (GVariant   *index,
+                          GVariant   *csum_bytes,
+                          OstreeObjectType objtype,
+                          guint64    *out_offset)
+{
+  gboolean ret = FALSE;
+  GVariant *index_contents;
+  gsize imax, imin;
+  gsize n;
+  guint32 target_objtype;
+
+  index_contents = g_variant_get_child_value (index, 2);
+
+  target_objtype = (guint32) objtype;
+
+  n = g_variant_n_children (index_contents);
+
+  if (n == 0)
+    goto out;
+
+  imax = n - 1;
+  imin = 0;
+  while (imax >= imin)
+    {
+      GVariant *cur_csum_bytes;
+      guint32 cur_objtype;
+      guint64 cur_offset;
+      gsize imid;
+      int c;
+
+      imid = (imin + imax) / 2;
+
+      g_variant_get_child (index_contents, imid, "(u@ayt)", &cur_objtype,
+                           &cur_csum_bytes, &cur_offset);      
+      cur_objtype = GUINT32_FROM_BE (cur_objtype);
+
+      c = ostree_cmp_checksum_bytes (cur_csum_bytes, csum_bytes);
+      if (c == 0)
+        {
+          if (cur_objtype < target_objtype)
+            c = -1;
+          else if (cur_objtype > target_objtype)
+            c = 1;
+        }
+      g_variant_unref (cur_csum_bytes);
+
+      if (c < 0)
+        imin = imid + 1;
+      else if (c > 0)
+        {
+          if (imid == 0)
+            goto out;
+          imax = imid - 1;
+        }
+      else
+        {
+          if (out_offset)
+            *out_offset = GUINT64_FROM_BE (cur_offset);
+          ret = TRUE;
+          goto out;
+        } 
+    }
+
+ out:
+  ot_clear_gvariant (&index_contents);
+  return ret;
+}
+
+gboolean
+ostree_validate_structureof_objtype (guint32    objtype,
+                                     GError   **error)
+{
+  objtype = GUINT32_FROM_BE (objtype);
+  if (objtype < OSTREE_OBJECT_TYPE_RAW_FILE 
+      || objtype > OSTREE_OBJECT_TYPE_COMMIT)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Invalid object type '%u'", objtype);
+      return FALSE;
+    }
+  return TRUE;
+}
+
+gboolean
+ostree_validate_structureof_checksum (GVariant  *checksum,
+                                      GError   **error)
+{
+  gsize n_children = g_variant_n_children (checksum);
+  if (n_children != 32)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Invalid checksum of length %" G_GUINT64_FORMAT
+                   " expected 32", (guint64) n_children);
+      return FALSE;
+    }
+  return TRUE;
+}
+
+static gboolean
+validate_variant (GVariant           *variant,
+                  const GVariantType *variant_type,
+                  GError            **error)
+{
+  if (!g_variant_is_normal_form (variant))
+    {
+      g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                           "Not normal form");
+      return FALSE;
+    }
+  if (!g_variant_is_of_type (variant, variant_type))
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Doesn't match variant type '%s'",
+                   (char*)variant_type);
+      return FALSE;
+    }
+  return TRUE;
+}
+
+gboolean
+ostree_validate_structureof_pack_index (GVariant      *index,
+                                        GError       **error)
+{
+  gboolean ret = FALSE;
+  const char *header;
+  GVariantIter *content_iter = NULL;
+  guint32 objtype;
+  GVariant *csum_bytes = NULL;
+  guint64 offset;
+
+  if (!validate_variant (index, OSTREE_PACK_INDEX_VARIANT_FORMAT, error))
+    goto out;
+
+  g_variant_get_child (index, 0, "&s", &header);
+
+  if (strcmp (header, "OSTv0PACKINDEX") != 0)
+    {
+      g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                           "Invalid pack index; doesn't match header");
+      goto out;
+    }
+
+  g_variant_get_child (index, 2, "a(uayt)", &content_iter);
+
+  while (g_variant_iter_loop (content_iter, "(u@ayt)",
+                              &objtype, &csum_bytes, &offset))
+    {
+      if (!ostree_validate_structureof_objtype (objtype, error))
+        goto out;
+      if (!ostree_validate_structureof_checksum (csum_bytes, error))
+        goto out;
+    }
+  csum_bytes = NULL;
+
+  ret = TRUE;
+ out:
+  if (content_iter)
+    g_variant_iter_free (content_iter);
+  ot_clear_gvariant (&csum_bytes);
+  return ret;
+}
+
+gboolean
+ostree_validate_structureof_pack_superindex (GVariant      *superindex,
+                                             GError       **error)
+{
+  gboolean ret = FALSE;
+  const char *header;
+  GVariant *csum_bytes = NULL;
+  GVariant *bloom = NULL;
+  GVariantIter *content_iter = NULL;
+
+  if (!validate_variant (superindex, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT, error))
+    goto out;
+
+  g_variant_get_child (superindex, 0, "&s", &header);
+
+  if (strcmp (header, "OSTv0SUPERPACKINDEX") != 0)
+    {
+      g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                           "Invalid pack superindex; doesn't match header");
+      goto out;
+    }
+
+  g_variant_get_child (superindex, 2, "a(ayay)", &content_iter);
+
+  while (g_variant_iter_loop (content_iter, "(@ay@ay)",
+                              &csum_bytes, &bloom))
+    {
+      if (!ostree_validate_structureof_checksum (csum_bytes, error))
+        goto out;
+    }
+  csum_bytes = NULL;
+
+  ret = TRUE;
+ out:
+  if (content_iter)
+    g_variant_iter_free (content_iter);
+  ot_clear_gvariant (&csum_bytes);
+  ot_clear_gvariant (&bloom);
+  return ret;
+}

src/libostree/ostree-core.h

@@ -97,9 +97,43 @@ typedef enum {
  */
 #define OSTREE_ARCHIVED_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(uuuuusa(ayay))")
 
+/* Pack super index
+ * s - OSTv0SUPERPACKINDEX
+ * a{sv} - Metadata
+ * a(say) - (pack file checksum, bloom filter)
+ */
+#define OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(ayay))")
+
+/* Pack index
+ * s - OSTv0PACKINDEX
+ * a{sv} - Metadata
+ * a(uayt) - (objtype, checksum, offset into packfile)
+ */
+#define OSTREE_PACK_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(uayt))")
+
+typedef enum {
+  OSTREE_PACK_FILE_ENTRY_FLAG_NONE = 0,
+  OSTREE_PACK_FILE_ENTRY_FLAG_GZIP = (1 << 0)
+} OstreePackFileEntryFlag;
+
+/* Pack files
+ * s - OSTv0PACKFILE
+ * a{sv} - Metadata
+ * t - number of entries
+ *
+ * Repeating pair of:
+ * <padding to alignment of 8>
+ * ( uyayay ) - objtype, flags, checksum, data
+ */
+#define OSTREE_PACK_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}t)")
+
+#define OSTREE_PACK_FILE_CONTENT_VARIANT_FORMAT G_VARIANT_TYPE ("(uyayay)")
+
 gboolean ostree_validate_checksum_string (const char *sha256,
                                           GError    **error);
 
+GVariant *ostree_checksum_to_bytes (const char *sha256);
+
 gboolean ostree_validate_rev (const char *rev, GError **error);
 
 void ostree_checksum_update_stat (GChecksum *checksum, guint32 uid, guint32 gid, guint32 mode);
@@ -108,6 +142,32 @@ const char * ostree_object_type_to_string (OstreeObjectType objtype);
 
 OstreeObjectType ostree_object_type_from_string (const char *str);
 
+guint ostree_hash_object_name (gconstpointer a);
+
+int ostree_cmp_checksum_bytes (GVariant *a, GVariant *b);
+
+GVariant *ostree_object_name_serialize (const char *checksum,
+                                        OstreeObjectType objtype);
+
+void ostree_object_name_deserialize (GVariant         *variant,
+                                     const char      **out_checksum,
+                                     OstreeObjectType *out_objtype);
+
+GVariant *ostree_object_name_serialize_v2 (const char *checksum,
+                                           OstreeObjectType objtype);
+
+void ostree_object_name_deserialize_v2_hex (GVariant         *variant,
+                                            char            **out_checksum,
+                                            OstreeObjectType *out_objtype);
+
+
+void ostree_object_name_deserialize_v2_bytes (GVariant         *variant,
+                                              const guchar    **out_checksum,
+                                              OstreeObjectType *out_objtype);
+
+GVariant * ostree_checksum_to_bytes (const char *sha256);
+char * ostree_checksum_from_bytes (GVariant *bytes);
+
 char * ostree_object_to_string (const char *checksum,
                                 OstreeObjectType objtype);
 
@@ -123,6 +183,11 @@ GVariant *ostree_get_xattrs_for_file (GFile       *f,
 
 GVariant *ostree_wrap_metadata_variant (OstreeObjectType type, GVariant *metadata);
 
+gboolean ostree_unwrap_metadata (GVariant              *container,
+                                 OstreeObjectType       expected_type,
+                                 GVariant             **out_variant,
+                                 GError               **error);
+
 gboolean ostree_set_xattrs (GFile *f, GVariant *xattrs,
                             GCancellable *cancellable, GError **error);
 
@@ -205,5 +270,40 @@ gboolean ostree_parse_archived_file_meta (GVariant         *data,
                                           GVariant        **out_xattrs,
                                           GError          **error);
 
+gboolean ostree_read_pack_entry_raw (guchar           *pack_data,
+                                     guint64           pack_len,
+                                     guint64           object_offset,
+                                     gboolean          trusted,
+                                     GVariant        **out_entry,
+                                     GCancellable     *cancellable,
+                                     GError          **error);
+
+GInputStream *ostree_read_pack_entry_as_stream (GVariant *pack_entry);
+
+gboolean ostree_read_pack_entry_variant (GVariant         *pack_entry,
+                                         OstreeObjectType  expected_objtype,
+                                         gboolean          trusted,
+                                         GVariant        **out_variant,
+                                         GCancellable     *cancellable,
+                                         GError          **error);
+
+gboolean ostree_pack_index_search (GVariant            *index,
+                                   GVariant           *csum_bytes,
+                                   OstreeObjectType    objtype,
+                                   guint64            *out_offset);
+
+/** VALIDATION **/
+
+gboolean ostree_validate_structureof_objtype (guint32    objtype,
+                                              GError   **error);
+
+gboolean ostree_validate_structureof_checksum (GVariant  *checksum,
+                                               GError   **error);
+
+gboolean ostree_validate_structureof_pack_index (GVariant      *index,
+                                                 GError       **error);
+
+gboolean ostree_validate_structureof_pack_superindex (GVariant      *superindex,
+                                                      GError       **error);
 
 #endif /* _OSTREE_REPO */

src/libostree/ostree-repo-file.c

@@ -724,39 +724,46 @@ bsearch_in_file_variant (GVariant  *variant,
                          const char *name,
                          int        *out_pos)
 {
-  int i, n;
-  int m;
+  gsize imax, imin;
+  gsize imid;
+  gsize n;
 
-  i = 0;
-  n = g_variant_n_children (variant) - 1;
-  m = 0;
+  n = g_variant_n_children (variant);
+  if (n == 0)
+    return FALSE;
 
-  while (i <= n)
+  imax = n - 1;
+  imin = 0;
+  while (imax >= imin)
     {
       GVariant *child;
       const char *cur;
       int cmp;
 
-      m = i + ((n - i) / 2);
+      imid = (imin + imax) / 2;
 
-      child = g_variant_get_child_value (variant, m);
+      child = g_variant_get_child_value (variant, imid);
       g_variant_get_child (child, 0, "&s", &cur, NULL);      
 
       cmp = strcmp (cur, name);
       if (cmp < 0)
-        i = m + 1;
+        imin = imid + 1;
       else if (cmp > 0)
-        n = m - 1;
+        {
+          if (imid == 0)
+            break;
+          imax = imid - 1;
+        }
       else
         {
           ot_clear_gvariant (&child);
-          *out_pos = m;
+          *out_pos = imid;
           return TRUE;
         }
       ot_clear_gvariant (&child);
     }
 
-  *out_pos = m;
+  *out_pos = imid;
   return FALSE;
 }
 
@@ -817,13 +824,9 @@ ostree_repo_file_tree_query_child (OstreeRepoFile  *self,
   const char *name = NULL;
   gboolean ret = FALSE;
   GFileInfo *ret_info = NULL;
-  GFile *archive_data_path = NULL;
-  GFileInfo *archive_data_info = NULL;
-  GVariant *archive_metadata = NULL;
   GVariant *files_variant = NULL;
   GVariant *dirs_variant = NULL;
   GVariant *tree_child_metadata = NULL;
-  GFile *local_child = NULL;
   GFileAttributeMatcher *matcher = NULL;
   int c;
 
@@ -844,40 +847,9 @@ ostree_repo_file_tree_query_child (OstreeRepoFile  *self,
 
       g_variant_get_child (files_variant, n, "(&s&s)", &name, &checksum);
 
-      local_child = ostree_repo_get_file_object_path (self->repo, checksum);
-
-      if (ostree_repo_get_mode (self->repo) == OSTREE_REPO_MODE_ARCHIVE)
-	{
-          if (!ostree_map_metadata_file (local_child, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META,
-                                         &archive_metadata, error))
-            goto out;
-          if (!ostree_parse_archived_file_meta (archive_metadata, &ret_info, NULL, error))
-            goto out;
-
-          archive_data_path = ostree_repo_get_object_path (self->repo, checksum,
-                                                           OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT);
-          archive_data_info = g_file_query_info (archive_data_path,
-                                                 OSTREE_GIO_FAST_QUERYINFO,
-                                                 G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
-                                                 cancellable,
-                                                 error);
-          if (!archive_data_info)
-            goto out;
-          
-          g_file_info_set_attribute_uint64 (ret_info, "standard::size",
-                                            g_file_info_get_attribute_uint64 (archive_data_info,
-                                                                              "standard::size"));
-	}
-      else
-	{
-          ret_info = g_file_query_info (local_child,
-                                        OSTREE_GIO_FAST_QUERYINFO,
-                                        G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
-                                        cancellable,
-                                        error);
-          if (!ret_info)
-            goto out;
-	}
+      if (!ostree_repo_load_file (self->repo, checksum, NULL, &ret_info, NULL,
+                                  cancellable, error))
+        goto out;
     }
   else
     {
@@ -918,12 +890,8 @@ ostree_repo_file_tree_query_child (OstreeRepoFile  *self,
   ot_transfer_out_value(out_info, &ret_info);
  out:
   g_clear_object (&ret_info);
-  g_clear_object (&local_child);
-  g_clear_object (&archive_data_path);
-  g_clear_object (&archive_data_info);
   if (matcher)
     g_file_attribute_matcher_unref (matcher);
-  ot_clear_gvariant (&archive_metadata);
   ot_clear_gvariant (&tree_child_metadata);
   ot_clear_gvariant (&files_variant);
   ot_clear_gvariant (&dirs_variant);

src/libostree/ostree-repo.h

@@ -97,6 +97,8 @@ gboolean      ostree_repo_find_object (OstreeRepo           *self,
                                        const char           *checksum,
                                        GFile               **out_stored_path,
                                        GFile               **out_pending_path,
+                                       char                **out_pack_checksum,
+                                       guint64              *out_pack_offset,
                                        GCancellable         *cancellable,
                                        GError              **error);
 
@@ -112,6 +114,7 @@ gboolean      ostree_repo_stage_object (OstreeRepo       *self,
 gboolean      ostree_repo_stage_object_trusted (OstreeRepo   *self,
                                                 OstreeObjectType objtype,
                                                 const char   *checksum,
+                                                gboolean          store_if_packed,
                                                 GFileInfo        *file_info,
                                                 GVariant         *xattrs,
                                                 GInputStream     *content,
@@ -141,6 +144,33 @@ gboolean      ostree_repo_load_variant (OstreeRepo  *self,
                                         GVariant     **out_variant,
                                         GError       **error);
 
+gboolean      ostree_repo_load_pack_index (OstreeRepo    *self,
+                                           const char    *sha256, 
+                                           GVariant     **out_variant,
+                                           GCancellable  *cancellable,
+                                           GError       **error);
+
+gboolean      ostree_repo_load_pack_data  (OstreeRepo    *self,
+                                           const char    *sha256,
+                                           guchar       **out_data,
+                                           GCancellable  *cancellable,
+                                           GError       **error);
+
+gboolean ostree_repo_map_pack_file (OstreeRepo    *self,
+                                    const char    *sha256,
+                                    guchar       **out_data,
+                                    guint64       *out_len,
+                                    GCancellable  *cancellable,
+                                    GError       **error);
+
+gboolean ostree_repo_load_file (OstreeRepo         *self,
+                                const char         *entry_sha256,
+                                GInputStream      **out_input,
+                                GFileInfo         **out_file_info,
+                                GVariant          **out_xattrs,
+                                GCancellable       *cancellable,
+                                GError            **error);
+
 typedef enum {
   OSTREE_REPO_COMMIT_FILTER_ALLOW,
   OSTREE_REPO_COMMIT_FILTER_SKIP
@@ -200,6 +230,53 @@ gboolean      ostree_repo_stage_commit (OstreeRepo   *self,
                                         GCancellable *cancellable,
                                         GError      **error);
 
+gboolean ostree_repo_regenerate_pack_index (OstreeRepo       *self,
+                                            GCancellable     *cancellable,
+                                            GError          **error);
+
+gboolean     ostree_repo_add_pack_file (OstreeRepo       *self,
+                                        const char       *checksum,
+                                        GFile            *pack_index_path,
+                                        GFile            *pack_data_path,
+                                        GCancellable     *cancellable,
+                                        GError          **error);
+
+gboolean     ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo       *self,
+                                                            const char       *remote_name,
+                                                            GFile            *superindex_path,
+                                                            GPtrArray       **out_cached_indexes,
+                                                            GPtrArray       **out_uncached_indexes,
+                                                            GCancellable     *cancellable,
+                                                            GError          **error);
+
+gboolean     ostree_repo_map_cached_remote_pack_index (OstreeRepo       *self,
+                                                       const char       *remote_name,
+                                                       const char       *pack_checksum,
+                                                       GVariant        **out_variant,
+                                                       GCancellable     *cancellable,
+                                                       GError          **error);
+
+gboolean     ostree_repo_add_cached_remote_pack_index (OstreeRepo       *self,
+                                                       const char       *remote_name,
+                                                       const char       *pack_checksum,
+                                                       GFile            *cached_path,
+                                                       GCancellable     *cancellable,
+                                                       GError          **error);
+
+gboolean     ostree_repo_get_cached_remote_pack_data (OstreeRepo       *self,
+                                                      const char       *remote_name,
+                                                      const char       *pack_checksum,
+                                                      GFile           **out_cached_path,
+                                                      GCancellable     *cancellable,
+                                                      GError          **error);
+
+gboolean     ostree_repo_take_cached_remote_pack_data (OstreeRepo       *self,
+                                                       const char       *remote_name,
+                                                       const char       *pack_checksum,
+                                                       GFile            *cached_path,
+                                                       GCancellable     *cancellable,
+                                                       GError          **error);
+
 typedef enum {
   OSTREE_REPO_CHECKOUT_MODE_NONE = 0,
   OSTREE_REPO_CHECKOUT_MODE_USER = 1
@@ -226,17 +303,36 @@ gboolean       ostree_repo_read_commit (OstreeRepo *self,
                                         GCancellable *cancellable,
                                         GError  **error);
 
-typedef void (*OstreeRepoObjectIter) (OstreeRepo *self, 
-                                      const char *checksum,
-                                      OstreeObjectType type,
-                                      GFile      *path,
-                                      GFileInfo  *fileinfo,
-                                      gpointer user_data);
+typedef enum {
+  OSTREE_REPO_LIST_OBJECTS_LOOSE = (1 << 0),
+  OSTREE_REPO_LIST_OBJECTS_PACKED = (1 << 1),
+  OSTREE_REPO_LIST_OBJECTS_ALL = (1 << 2)
+} OstreeRepoListObjectsFlags;
 
-gboolean     ostree_repo_iter_objects (OstreeRepo  *self,
-                                       OstreeRepoObjectIter callback,
-                                       gpointer       user_data,
-                                       GError        **error);
+/**
+ * OSTREE_REPO_LIST_OBJECTS_VARIANT_TYPE:
+ *
+ * b - %TRUE if object is available "loose"
+ * as - List of pack file checksums in which this object appears
+ */
+#define OSTREE_REPO_LIST_OBJECTS_VARIANT_TYPE (G_VARIANT_TYPE ("(bas)")
+
+gboolean ostree_repo_list_objects (OstreeRepo                  *self,
+                                   OstreeRepoListObjectsFlags   flags,
+                                   GHashTable                 **out_objects,
+                                   GCancellable                *cancellable,
+                                   GError                     **error);
+
+gboolean ostree_repo_list_pack_indexes (OstreeRepo              *self,
+                                        GPtrArray              **out_indexes,
+                                        GCancellable            *cancellable,
+                                        GError                 **error);
+
+GFile * ostree_repo_get_pack_index_path (OstreeRepo         *self,
+                                         const char         *checksum);
+
+GFile * ostree_repo_get_pack_data_path (OstreeRepo         *self,
+                                        const char         *checksum);
 
 G_END_DECLS