diff --git a/Makefile-ostree.am b/Makefile-ostree.am index 83333ebb..0eedd140 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -34,6 +34,8 @@ ostree_SOURCES = src/ostree/main.c \ src/ostree/ot-builtin-ls.c \ src/ostree/ot-builtin-prune.c \ src/ostree/ot-builtin-remote.c \ + src/ostree/ot-builtin-pack.c \ + src/ostree/ot-builtin-unpack.c \ src/ostree/ot-builtin-rev-parse.c \ src/ostree/ot-builtin-show.c \ src/ostree/ot-main.h \ diff --git a/src/libostree/README.md b/src/libostree/README.md new file mode 100644 index 00000000..83d9f5fb --- /dev/null +++ b/src/libostree/README.md @@ -0,0 +1,78 @@ +Repository design +----------------- + +At the heart of OSTree is the repository. It's very similar to git, +with the idea of content-addressed storage. However, OSTree is +designed to store operating system binaries, not source code. There +are several consequences to this. The key difference as compared to +git is that the OSTree definition of "content" includes key Unix +metadata such as owner uid/gid, as well as all extended attributes. + +Essentially OSTree is designed so that if two files have the same +OSTree checksum, it's safe to replace them with a hard link. This +fundamental design means that an OSTree repository imposes negligible +overhead. In contrast, a git repository stores copies of +zlib-compressed data. + +Key differences versus git +-------------------------- + + * As mentioned above, extended attributes and owner uid/gid are versioned + * Optimized for Unix hardlinks between repository and checkout + * SHA256 instead of SHA1 + * Support for empty directories + +Binary files +------------ + +While this is still in planning, I plan to heavily optimize OSTree for +versioning ELF operating systems. In industry jargon, this would be +"content-aware storage". + +Trimming history +---------------- + +OSTree will also be optimized to trim intermediate history; in theory +one can regenerate binaries from corresponding (git) source code, so +we don't need to keep all possible builds over time. + +MILESTONE 1 +----------- +* Basic pack files (like git) + +MILESTONE 2 +----------- +* Store checksums as ay +* Drop version/metadata from tree/dirmeta objects +* Split pack files into metadata/data +* Restructure repository so that links can be generated as a cache; + i.e. objects/raw, pack files are now the canonical +* For files, checksum combination of metadata variant + raw data + - i.e. there is only OSTREE_OBJECT_TYPE_FILE (again) + +MILESTONE 3 +----------- + +* Drop archive/raw distinction - archive repositories always generate + packfiles per commit +* Include git packv4 ideas: + - metadata packfiles have string dictionary (tree filenames and checksums) + - data packfiles match up similar objects +* Rolling checksums for partitioning large files? Kernel debuginfo +* Improved pack clustering + - file fingerprinting? +* ELF-x86 aware deltas + +Related work in storage +----------------------- + +git: http://git-scm.com/ +Venti: http://plan9.bell-labs.com/magic/man2html/6/venti +Elephant FS: http://www.hpl.hp.com/personal/Alistair_Veitch/papers/elephant-hotos/index.html + +Compression +----------- + +xdelta: http://xdelta.org/ +Bsdiff: http://www.daemonology.net/bsdiff/ +xz: http://tukaani.org/xz/ diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 263988cf..1e3a9e1e 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -28,6 +28,9 @@ #include #include +#define ALIGN_VALUE(this, boundary) \ + (( ((unsigned long)(this)) + (((unsigned long)(boundary)) -1)) & (~(((unsigned long)(boundary))-1))) + gboolean ostree_validate_checksum_string (const char *sha256, GError **error) @@ -486,6 +489,34 @@ ostree_set_xattrs (GFile *f, return ret; } +gboolean +ostree_unwrap_metadata (GVariant *container, + OstreeObjectType expected_type, + GVariant **out_variant, + GError **error) +{ + gboolean ret = FALSE; + GVariant *ret_variant = NULL; + guint32 actual_type; + + g_variant_get (container, "(uv)", + &actual_type, &ret_variant); + actual_type = GUINT32_FROM_BE (actual_type); + if (actual_type != expected_type) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Corrupted metadata object; found type %u, expected %u", + actual_type, (guint32)expected_type); + goto out; + } + + ret = TRUE; + ot_transfer_out_value (out_variant, &ret_variant); + out: + ot_clear_gvariant (&ret_variant); + return ret; +} + gboolean ostree_map_metadata_file (GFile *file, OstreeObjectType expected_type, @@ -495,22 +526,16 @@ ostree_map_metadata_file (GFile *file, gboolean ret = FALSE; GVariant *ret_variant = NULL; GVariant *container = NULL; - guint32 actual_type; if (!ot_util_variant_map (file, OSTREE_SERIALIZED_VARIANT_FORMAT, &container, error)) goto out; - g_variant_get (container, "(uv)", - &actual_type, &ret_variant); - ot_util_variant_take_ref (ret_variant); - actual_type = GUINT32_FROM_BE (actual_type); - if (actual_type != expected_type) + if (!ostree_unwrap_metadata (container, expected_type, &ret_variant, + error)) { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Corrupted metadata object '%s'; found type %u, expected %u", - ot_gfile_get_path_cached (file), - actual_type, (guint32)expected_type); + g_prefix_error (error, "While parsing '%s': ", + ot_gfile_get_path_cached (file)); goto out; } @@ -584,6 +609,142 @@ ostree_object_from_string (const char *str, *out_objtype = ostree_object_type_from_string (dot + 1); } +guint +ostree_hash_object_name (gconstpointer a) +{ + GVariant *variant = (gpointer)a; + const char *checksum; + OstreeObjectType objtype; + gint objtype_int; + + ostree_object_name_deserialize (variant, &checksum, &objtype); + objtype_int = (gint) objtype; + return g_str_hash (checksum) + g_int_hash (&objtype_int); +} + +int +ostree_cmp_checksum_bytes (GVariant *a, + GVariant *b) +{ + gconstpointer a_data; + gconstpointer b_data; + gsize a_n_elts; + gsize b_n_elts; + + a_data = g_variant_get_fixed_array (a, &a_n_elts, 1); + g_assert (a_n_elts == 32); + b_data = g_variant_get_fixed_array (b, &b_n_elts, 1); + g_assert (b_n_elts == 32); + + return memcmp (a_data, b_data, 32); +} + + +GVariant * +ostree_object_name_serialize (const char *checksum, + OstreeObjectType objtype) +{ + return g_variant_new ("(su)", checksum, (guint32)objtype); +} + +void +ostree_object_name_deserialize (GVariant *variant, + const char **out_checksum, + OstreeObjectType *out_objtype) +{ + guint32 objtype_u32; + g_variant_get (variant, "(&su)", out_checksum, &objtype_u32); + *out_objtype = (OstreeObjectType)objtype_u32; +} + +GVariant * +ostree_checksum_to_bytes (const char *sha256) +{ + guchar result[32]; + guint i; + guint j; + + for (i = 0, j = 0; i < 32; i += 1, j += 2) + { + gint big, little; + + g_assert (sha256[j]); + g_assert (sha256[j+1]); + + big = g_ascii_xdigit_value (sha256[j]); + little = g_ascii_xdigit_value (sha256[j+1]); + + g_assert (big != -1); + g_assert (little != -1); + + result[i] = (big << 4) | little; + } + + return g_variant_new_fixed_array (G_VARIANT_TYPE ("y"), + (guchar*)result, 32, 1); +} + +char * +ostree_checksum_from_bytes (GVariant *csum_bytes) +{ + static const gchar hexchars[] = "0123456789abcdef"; + char *ret; + const guchar *bytes; + gsize n_elts; + guint i, j; + + bytes = g_variant_get_fixed_array (csum_bytes, &n_elts, 1); + g_assert (n_elts == 32); + + ret = g_malloc (65); + + for (i = 0, j = 0; i < 32; i++, j += 2) + { + guchar byte = bytes[i]; + ret[j] = hexchars[byte >> 4]; + ret[j+1] = hexchars[byte & 0xF]; + } + ret[j] = '\0'; + + return ret; +} + +GVariant * +ostree_object_name_serialize_v2 (const char *checksum, + OstreeObjectType objtype) +{ + return g_variant_new ("(u@ay)", (guint32)objtype, ostree_checksum_to_bytes (checksum)); +} + +void +ostree_object_name_deserialize_v2_hex (GVariant *variant, + char **out_checksum, + OstreeObjectType *out_objtype) +{ + GVariant *csum_bytes; + guint32 objtype_u32; + + g_variant_get (variant, "(u@ay)", &objtype_u32, &csum_bytes); + g_variant_ref_sink (csum_bytes); + *out_checksum = ostree_checksum_from_bytes (csum_bytes); + g_variant_unref (csum_bytes); + *out_objtype = (OstreeObjectType)objtype_u32; +} + +void +ostree_object_name_deserialize_v2_bytes (GVariant *variant, + const guchar **out_checksum, + OstreeObjectType *out_objtype) +{ + GVariant *csum_bytes; + guint32 objtype_u32; + gsize n_elts; + + g_variant_get (variant, "(u@ay)", &objtype_u32, &csum_bytes); + *out_checksum = (guchar*)g_variant_get_fixed_array (csum_bytes, &n_elts, 1); + *out_objtype = (OstreeObjectType)objtype_u32; +} + char * ostree_get_relative_object_path (const char *checksum, OstreeObjectType type) @@ -1074,3 +1235,350 @@ ostree_create_temp_hardlink (GFile *dir, g_clear_object (&possible_file); return ret; } + +gboolean +ostree_read_pack_entry_raw (guchar *pack_data, + guint64 pack_len, + guint64 offset, + gboolean trusted, + GVariant **out_entry, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GVariant *ret_entry = NULL; + guint64 entry_start; + guint64 entry_end; + guint32 entry_len; + + if (G_UNLIKELY (!(offset <= pack_len))) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Corrupted pack index; out of range offset %" G_GUINT64_FORMAT, + offset); + goto out; + } + if (G_UNLIKELY (!((offset & 0x3) == 0))) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Corrupted pack index; unaligned offset %" G_GUINT64_FORMAT, + offset); + goto out; + } + + entry_start = ALIGN_VALUE (offset + 4, 8); + if (G_UNLIKELY (!(entry_start <= pack_len))) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Corrupted pack index; out of range data offset %" G_GUINT64_FORMAT, + entry_start); + goto out; + } + + g_assert ((((guint64)pack_data+offset) & 0x3) == 0); + entry_len = GUINT32_FROM_BE (*((guint32*)(pack_data+offset))); + + entry_end = entry_start + entry_len; + if (G_UNLIKELY (!(entry_end <= pack_len))) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Corrupted pack index; out of range entry length %u", + entry_len); + goto out; + } + + ret_entry = g_variant_new_from_data (OSTREE_PACK_FILE_CONTENT_VARIANT_FORMAT, + pack_data+entry_start, entry_len, + trusted, NULL, NULL); + ret = TRUE; + ot_transfer_out_value (out_entry, &ret_entry); + out: + ot_clear_gvariant (&ret_entry); + return ret; +} + +GInputStream * +ostree_read_pack_entry_as_stream (GVariant *pack_entry) +{ + GInputStream *memory_input; + GInputStream *ret_input = NULL; + GVariant *pack_data = NULL; + guchar entry_flags; + gconstpointer data_ptr; + gsize data_len; + + g_variant_get_child (pack_entry, 1, "y", &entry_flags); + g_variant_get_child (pack_entry, 3, "@ay", &pack_data); + + data_ptr = g_variant_get_fixed_array (pack_data, &data_len, 1); + memory_input = g_memory_input_stream_new_from_data (data_ptr, data_len, NULL); + g_object_set_data_full ((GObject*)memory_input, "ostree-mem-gvariant", + pack_data, (GDestroyNotify) g_variant_unref); + + if (entry_flags & OSTREE_PACK_FILE_ENTRY_FLAG_GZIP) + { + GConverter *decompressor; + + decompressor = (GConverter*)g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP); + ret_input = (GInputStream*)g_object_new (G_TYPE_CONVERTER_INPUT_STREAM, + "converter", decompressor, + "base-stream", memory_input, + "close-base-stream", TRUE, + NULL); + g_object_unref (decompressor); + } + else + { + ret_input = memory_input; + memory_input = NULL; + } + + return ret_input; +} + +gboolean +ostree_read_pack_entry_variant (GVariant *pack_entry, + OstreeObjectType expected_objtype, + gboolean trusted, + GVariant **out_variant, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GInputStream *stream = NULL; + GVariant *container_variant = NULL; + GVariant *ret_variant = NULL; + guint32 actual_type; + + stream = ostree_read_pack_entry_as_stream (pack_entry); + + if (!ot_util_variant_from_stream (stream, OSTREE_SERIALIZED_VARIANT_FORMAT, + trusted, &container_variant, cancellable, error)) + goto out; + + g_variant_ref_sink (container_variant); + + g_variant_get (container_variant, "(uv)", + &actual_type, &ret_variant); + actual_type = GUINT32_FROM_BE (actual_type); + + if (actual_type != expected_objtype) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Corrupted metadata object in pack file; found type %u, expected %u", + actual_type, (guint32)expected_objtype); + goto out; + } + + ret = TRUE; + ot_transfer_out_value (out_variant, &ret_variant); + out: + g_clear_object (&stream); + ot_clear_gvariant (&ret_variant); + ot_clear_gvariant (&container_variant); + return ret; +} + +gboolean +ostree_pack_index_search (GVariant *index, + GVariant *csum_bytes, + OstreeObjectType objtype, + guint64 *out_offset) +{ + gboolean ret = FALSE; + GVariant *index_contents; + gsize imax, imin; + gsize n; + guint32 target_objtype; + + index_contents = g_variant_get_child_value (index, 2); + + target_objtype = (guint32) objtype; + + n = g_variant_n_children (index_contents); + + if (n == 0) + goto out; + + imax = n - 1; + imin = 0; + while (imax >= imin) + { + GVariant *cur_csum_bytes; + guint32 cur_objtype; + guint64 cur_offset; + gsize imid; + int c; + + imid = (imin + imax) / 2; + + g_variant_get_child (index_contents, imid, "(u@ayt)", &cur_objtype, + &cur_csum_bytes, &cur_offset); + cur_objtype = GUINT32_FROM_BE (cur_objtype); + + c = ostree_cmp_checksum_bytes (cur_csum_bytes, csum_bytes); + if (c == 0) + { + if (cur_objtype < target_objtype) + c = -1; + else if (cur_objtype > target_objtype) + c = 1; + } + g_variant_unref (cur_csum_bytes); + + if (c < 0) + imin = imid + 1; + else if (c > 0) + { + if (imid == 0) + goto out; + imax = imid - 1; + } + else + { + if (out_offset) + *out_offset = GUINT64_FROM_BE (cur_offset); + ret = TRUE; + goto out; + } + } + + out: + ot_clear_gvariant (&index_contents); + return ret; +} + +gboolean +ostree_validate_structureof_objtype (guint32 objtype, + GError **error) +{ + objtype = GUINT32_FROM_BE (objtype); + if (objtype < OSTREE_OBJECT_TYPE_RAW_FILE + || objtype > OSTREE_OBJECT_TYPE_COMMIT) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid object type '%u'", objtype); + return FALSE; + } + return TRUE; +} + +gboolean +ostree_validate_structureof_checksum (GVariant *checksum, + GError **error) +{ + gsize n_children = g_variant_n_children (checksum); + if (n_children != 32) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid checksum of length %" G_GUINT64_FORMAT + " expected 32", (guint64) n_children); + return FALSE; + } + return TRUE; +} + +static gboolean +validate_variant (GVariant *variant, + const GVariantType *variant_type, + GError **error) +{ + if (!g_variant_is_normal_form (variant)) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Not normal form"); + return FALSE; + } + if (!g_variant_is_of_type (variant, variant_type)) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Doesn't match variant type '%s'", + (char*)variant_type); + return FALSE; + } + return TRUE; +} + +gboolean +ostree_validate_structureof_pack_index (GVariant *index, + GError **error) +{ + gboolean ret = FALSE; + const char *header; + GVariantIter *content_iter = NULL; + guint32 objtype; + GVariant *csum_bytes = NULL; + guint64 offset; + + if (!validate_variant (index, OSTREE_PACK_INDEX_VARIANT_FORMAT, error)) + goto out; + + g_variant_get_child (index, 0, "&s", &header); + + if (strcmp (header, "OSTv0PACKINDEX") != 0) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid pack index; doesn't match header"); + goto out; + } + + g_variant_get_child (index, 2, "a(uayt)", &content_iter); + + while (g_variant_iter_loop (content_iter, "(u@ayt)", + &objtype, &csum_bytes, &offset)) + { + if (!ostree_validate_structureof_objtype (objtype, error)) + goto out; + if (!ostree_validate_structureof_checksum (csum_bytes, error)) + goto out; + } + csum_bytes = NULL; + + ret = TRUE; + out: + if (content_iter) + g_variant_iter_free (content_iter); + ot_clear_gvariant (&csum_bytes); + return ret; +} + +gboolean +ostree_validate_structureof_pack_superindex (GVariant *superindex, + GError **error) +{ + gboolean ret = FALSE; + const char *header; + GVariant *csum_bytes = NULL; + GVariant *bloom = NULL; + GVariantIter *content_iter = NULL; + + if (!validate_variant (superindex, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT, error)) + goto out; + + g_variant_get_child (superindex, 0, "&s", &header); + + if (strcmp (header, "OSTv0SUPERPACKINDEX") != 0) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid pack superindex; doesn't match header"); + goto out; + } + + g_variant_get_child (superindex, 2, "a(ayay)", &content_iter); + + while (g_variant_iter_loop (content_iter, "(@ay@ay)", + &csum_bytes, &bloom)) + { + if (!ostree_validate_structureof_checksum (csum_bytes, error)) + goto out; + } + csum_bytes = NULL; + + ret = TRUE; + out: + if (content_iter) + g_variant_iter_free (content_iter); + ot_clear_gvariant (&csum_bytes); + ot_clear_gvariant (&bloom); + return ret; +} diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index a3abb3ff..e733f512 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -97,9 +97,43 @@ typedef enum { */ #define OSTREE_ARCHIVED_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(uuuuusa(ayay))") +/* Pack super index + * s - OSTv0SUPERPACKINDEX + * a{sv} - Metadata + * a(say) - (pack file checksum, bloom filter) + */ +#define OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(ayay))") + +/* Pack index + * s - OSTv0PACKINDEX + * a{sv} - Metadata + * a(uayt) - (objtype, checksum, offset into packfile) + */ +#define OSTREE_PACK_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(uayt))") + +typedef enum { + OSTREE_PACK_FILE_ENTRY_FLAG_NONE = 0, + OSTREE_PACK_FILE_ENTRY_FLAG_GZIP = (1 << 0) +} OstreePackFileEntryFlag; + +/* Pack files + * s - OSTv0PACKFILE + * a{sv} - Metadata + * t - number of entries + * + * Repeating pair of: + * + * ( uyayay ) - objtype, flags, checksum, data + */ +#define OSTREE_PACK_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}t)") + +#define OSTREE_PACK_FILE_CONTENT_VARIANT_FORMAT G_VARIANT_TYPE ("(uyayay)") + gboolean ostree_validate_checksum_string (const char *sha256, GError **error); +GVariant *ostree_checksum_to_bytes (const char *sha256); + gboolean ostree_validate_rev (const char *rev, GError **error); void ostree_checksum_update_stat (GChecksum *checksum, guint32 uid, guint32 gid, guint32 mode); @@ -108,6 +142,32 @@ const char * ostree_object_type_to_string (OstreeObjectType objtype); OstreeObjectType ostree_object_type_from_string (const char *str); +guint ostree_hash_object_name (gconstpointer a); + +int ostree_cmp_checksum_bytes (GVariant *a, GVariant *b); + +GVariant *ostree_object_name_serialize (const char *checksum, + OstreeObjectType objtype); + +void ostree_object_name_deserialize (GVariant *variant, + const char **out_checksum, + OstreeObjectType *out_objtype); + +GVariant *ostree_object_name_serialize_v2 (const char *checksum, + OstreeObjectType objtype); + +void ostree_object_name_deserialize_v2_hex (GVariant *variant, + char **out_checksum, + OstreeObjectType *out_objtype); + + +void ostree_object_name_deserialize_v2_bytes (GVariant *variant, + const guchar **out_checksum, + OstreeObjectType *out_objtype); + +GVariant * ostree_checksum_to_bytes (const char *sha256); +char * ostree_checksum_from_bytes (GVariant *bytes); + char * ostree_object_to_string (const char *checksum, OstreeObjectType objtype); @@ -123,6 +183,11 @@ GVariant *ostree_get_xattrs_for_file (GFile *f, GVariant *ostree_wrap_metadata_variant (OstreeObjectType type, GVariant *metadata); +gboolean ostree_unwrap_metadata (GVariant *container, + OstreeObjectType expected_type, + GVariant **out_variant, + GError **error); + gboolean ostree_set_xattrs (GFile *f, GVariant *xattrs, GCancellable *cancellable, GError **error); @@ -205,5 +270,40 @@ gboolean ostree_parse_archived_file_meta (GVariant *data, GVariant **out_xattrs, GError **error); +gboolean ostree_read_pack_entry_raw (guchar *pack_data, + guint64 pack_len, + guint64 object_offset, + gboolean trusted, + GVariant **out_entry, + GCancellable *cancellable, + GError **error); + +GInputStream *ostree_read_pack_entry_as_stream (GVariant *pack_entry); + +gboolean ostree_read_pack_entry_variant (GVariant *pack_entry, + OstreeObjectType expected_objtype, + gboolean trusted, + GVariant **out_variant, + GCancellable *cancellable, + GError **error); + +gboolean ostree_pack_index_search (GVariant *index, + GVariant *csum_bytes, + OstreeObjectType objtype, + guint64 *out_offset); + +/** VALIDATION **/ + +gboolean ostree_validate_structureof_objtype (guint32 objtype, + GError **error); + +gboolean ostree_validate_structureof_checksum (GVariant *checksum, + GError **error); + +gboolean ostree_validate_structureof_pack_index (GVariant *index, + GError **error); + +gboolean ostree_validate_structureof_pack_superindex (GVariant *superindex, + GError **error); #endif /* _OSTREE_REPO */ diff --git a/src/libostree/ostree-repo-file.c b/src/libostree/ostree-repo-file.c index 9cc7b823..ac4b1292 100644 --- a/src/libostree/ostree-repo-file.c +++ b/src/libostree/ostree-repo-file.c @@ -724,39 +724,46 @@ bsearch_in_file_variant (GVariant *variant, const char *name, int *out_pos) { - int i, n; - int m; + gsize imax, imin; + gsize imid; + gsize n; - i = 0; - n = g_variant_n_children (variant) - 1; - m = 0; + n = g_variant_n_children (variant); + if (n == 0) + return FALSE; - while (i <= n) + imax = n - 1; + imin = 0; + while (imax >= imin) { GVariant *child; const char *cur; int cmp; - m = i + ((n - i) / 2); + imid = (imin + imax) / 2; - child = g_variant_get_child_value (variant, m); + child = g_variant_get_child_value (variant, imid); g_variant_get_child (child, 0, "&s", &cur, NULL); cmp = strcmp (cur, name); if (cmp < 0) - i = m + 1; + imin = imid + 1; else if (cmp > 0) - n = m - 1; + { + if (imid == 0) + break; + imax = imid - 1; + } else { ot_clear_gvariant (&child); - *out_pos = m; + *out_pos = imid; return TRUE; } ot_clear_gvariant (&child); } - *out_pos = m; + *out_pos = imid; return FALSE; } @@ -817,13 +824,9 @@ ostree_repo_file_tree_query_child (OstreeRepoFile *self, const char *name = NULL; gboolean ret = FALSE; GFileInfo *ret_info = NULL; - GFile *archive_data_path = NULL; - GFileInfo *archive_data_info = NULL; - GVariant *archive_metadata = NULL; GVariant *files_variant = NULL; GVariant *dirs_variant = NULL; GVariant *tree_child_metadata = NULL; - GFile *local_child = NULL; GFileAttributeMatcher *matcher = NULL; int c; @@ -844,40 +847,9 @@ ostree_repo_file_tree_query_child (OstreeRepoFile *self, g_variant_get_child (files_variant, n, "(&s&s)", &name, &checksum); - local_child = ostree_repo_get_file_object_path (self->repo, checksum); - - if (ostree_repo_get_mode (self->repo) == OSTREE_REPO_MODE_ARCHIVE) - { - if (!ostree_map_metadata_file (local_child, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, - &archive_metadata, error)) - goto out; - if (!ostree_parse_archived_file_meta (archive_metadata, &ret_info, NULL, error)) - goto out; - - archive_data_path = ostree_repo_get_object_path (self->repo, checksum, - OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT); - archive_data_info = g_file_query_info (archive_data_path, - OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, - error); - if (!archive_data_info) - goto out; - - g_file_info_set_attribute_uint64 (ret_info, "standard::size", - g_file_info_get_attribute_uint64 (archive_data_info, - "standard::size")); - } - else - { - ret_info = g_file_query_info (local_child, - OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, - error); - if (!ret_info) - goto out; - } + if (!ostree_repo_load_file (self->repo, checksum, NULL, &ret_info, NULL, + cancellable, error)) + goto out; } else { @@ -918,12 +890,8 @@ ostree_repo_file_tree_query_child (OstreeRepoFile *self, ot_transfer_out_value(out_info, &ret_info); out: g_clear_object (&ret_info); - g_clear_object (&local_child); - g_clear_object (&archive_data_path); - g_clear_object (&archive_data_info); if (matcher) g_file_attribute_matcher_unref (matcher); - ot_clear_gvariant (&archive_metadata); ot_clear_gvariant (&tree_child_metadata); ot_clear_gvariant (&files_variant); ot_clear_gvariant (&dirs_variant); diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 4c99d88c..14618697 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -60,6 +60,8 @@ struct _OstreeRepoPrivate { GFile *local_heads_dir; GFile *remote_heads_dir; GFile *objects_dir; + GFile *pack_dir; + GFile *remote_cache_dir; GFile *config_file; gboolean inited; @@ -68,6 +70,9 @@ struct _OstreeRepoPrivate { GKeyFile *config; OstreeRepoMode mode; + GHashTable *pack_index_mappings; + GHashTable *pack_data_mappings; + GHashTable *pending_transaction; }; @@ -83,7 +88,11 @@ ostree_repo_finalize (GObject *object) g_clear_object (&priv->local_heads_dir); g_clear_object (&priv->remote_heads_dir); g_clear_object (&priv->objects_dir); + g_clear_object (&priv->pack_dir); + g_clear_object (&priv->remote_cache_dir); g_clear_object (&priv->config_file); + g_hash_table_destroy (priv->pack_index_mappings); + g_hash_table_destroy (priv->pack_data_mappings); g_hash_table_destroy (priv->pending_transaction); if (priv->config) g_key_file_free (priv->config); @@ -154,6 +163,8 @@ ostree_repo_constructor (GType gtype, priv->remote_heads_dir = g_file_resolve_relative_path (priv->repodir, "refs/remotes"); priv->objects_dir = g_file_get_child (priv->repodir, "objects"); + priv->pack_dir = g_file_get_child (priv->objects_dir, "pack"); + priv->remote_cache_dir = g_file_get_child (priv->repodir, "remote-cache"); priv->config_file = g_file_get_child (priv->repodir, "config"); return object; @@ -185,6 +196,12 @@ ostree_repo_init (OstreeRepo *self) { OstreeRepoPrivate *priv = GET_PRIVATE (self); + priv->pack_index_mappings = g_hash_table_new_full (g_str_hash, g_str_equal, + g_free, + (GDestroyNotify)g_variant_unref); + priv->pack_data_mappings = g_hash_table_new_full (g_str_hash, g_str_equal, + g_free, + (GDestroyNotify)g_mapped_file_unref); priv->pending_transaction = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); @@ -335,7 +352,6 @@ ostree_repo_resolve_rev (OstreeRepo *self, GFile *origindir = NULL; GError *temp_error = NULL; GVariant *commit = NULL; - GPtrArray *components = NULL; g_return_val_if_fail (rev != NULL, FALSE); @@ -768,48 +784,6 @@ get_pending_object_path (OstreeRepo *self, return ret; } -gboolean -ostree_repo_find_object (OstreeRepo *self, - OstreeObjectType objtype, - const char *checksum, - GFile **out_stored_path, - GFile **out_pending_path, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - GFile *object_path = NULL; - struct stat stbuf; - - g_return_val_if_fail (out_stored_path, FALSE); - g_return_val_if_fail (out_pending_path, FALSE); - - object_path = ostree_repo_get_object_path (self, checksum, objtype); - - *out_stored_path = NULL; - *out_pending_path = NULL; - if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0) - { - *out_stored_path = object_path; - object_path = NULL; - } - else - { - g_clear_object (&object_path); - object_path = get_pending_object_path (self, checksum, objtype); - if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0) - { - *out_pending_path = object_path; - object_path = NULL; - } - } - - ret = TRUE; - /* out: */ - g_clear_object (&object_path); - return ret; -} - static GFileInfo * dup_file_info_owned_by_me (GFileInfo *file_info) { @@ -824,6 +798,7 @@ dup_file_info_owned_by_me (GFileInfo *file_info) static gboolean stage_object_impl (OstreeRepo *self, OstreeObjectType objtype, + gboolean store_if_packed, GFileInfo *file_info, GVariant *xattrs, GInputStream *input, @@ -941,7 +916,7 @@ impl_stage_archive_file_object_from_raw (OstreeRepo *self, g_checksum_update (ret_checksum, (guint8*)g_variant_get_data (xattrs), g_variant_get_size (xattrs)); } - if (expected_checksum) + if (expected_checksum && ret_checksum) { if (strcmp (g_checksum_get_string (ret_checksum), expected_checksum) != 0) { @@ -953,6 +928,8 @@ impl_stage_archive_file_object_from_raw (OstreeRepo *self, } actual_checksum = expected_checksum; } + else if (expected_checksum) + actual_checksum = expected_checksum; else actual_checksum = g_checksum_get_string (ret_checksum); @@ -980,6 +957,7 @@ impl_stage_archive_file_object_from_raw (OstreeRepo *self, static gboolean stage_object_impl (OstreeRepo *self, OstreeObjectType objtype, + gboolean store_if_packed, GFileInfo *file_info, GVariant *xattrs, GInputStream *input, @@ -995,6 +973,8 @@ stage_object_impl (OstreeRepo *self, GFile *temp_file = NULL; GFile *stored_path = NULL; GFile *pending_path = NULL; + char *pack_checksum = NULL; + guint64 pack_offset; const char *actual_checksum; g_return_val_if_fail (priv->in_transaction, FALSE); @@ -1006,15 +986,28 @@ stage_object_impl (OstreeRepo *self, if (expected_checksum) { - if (!ostree_repo_find_object (self, objtype, expected_checksum, &stored_path, &pending_path, - cancellable, error)) - goto out; + if (!store_if_packed) + { + if (!ostree_repo_find_object (self, objtype, expected_checksum, + &stored_path, &pending_path, + &pack_checksum, &pack_offset, + cancellable, error)) + goto out; + } + else + { + if (!ostree_repo_find_object (self, objtype, expected_checksum, + &stored_path, &pending_path, + NULL, NULL, + cancellable, error)) + goto out; + } } g_assert (objtype != OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT); g_assert (objtype != OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META); - if (stored_path == NULL && pending_path == NULL) + if (stored_path == NULL && pending_path == NULL && pack_checksum == NULL) { if (objtype == OSTREE_OBJECT_TYPE_RAW_FILE) { @@ -1075,7 +1068,7 @@ stage_object_impl (OstreeRepo *self, } else { - g_assert (stored_path != NULL); + g_assert (stored_path != NULL || pack_checksum != NULL); /* Nothing to do */ } @@ -1088,6 +1081,7 @@ stage_object_impl (OstreeRepo *self, g_clear_object (&temp_info); g_clear_object (&stored_path); g_clear_object (&pending_path); + g_free (pack_checksum); ot_clear_checksum (&ret_checksum); return ret; } @@ -1219,7 +1213,7 @@ stage_gvariant_object (OstreeRepo *self, g_variant_get_size (serialized), NULL); - if (!stage_object_impl (self, type, + if (!stage_object_impl (self, type, FALSE, NULL, NULL, mem, NULL, &ret_checksum, cancellable, error)) goto out; @@ -1233,33 +1227,6 @@ stage_gvariant_object (OstreeRepo *self, return ret; } -gboolean -ostree_repo_load_variant (OstreeRepo *self, - OstreeObjectType expected_type, - const char *sha256, - GVariant **out_variant, - GError **error) -{ - gboolean ret = FALSE; - GFile *object_path = NULL; - GFile *tmpfile = NULL; - GVariant *ret_variant = NULL; - - g_return_val_if_fail (OSTREE_OBJECT_TYPE_IS_META (expected_type), FALSE); - - object_path = ostree_repo_get_object_path (self, sha256, expected_type); - if (!ostree_map_metadata_file (object_path, expected_type, &ret_variant, error)) - goto out; - - ret = TRUE; - ot_transfer_out_value (out_variant, &ret_variant); - out: - g_clear_object (&object_path); - g_clear_object (&tmpfile); - ot_clear_gvariant (&ret_variant); - return ret; -} - static gboolean stage_directory_meta (OstreeRepo *self, GFileInfo *file_info, @@ -1309,13 +1276,14 @@ gboolean ostree_repo_stage_object_trusted (OstreeRepo *self, OstreeObjectType objtype, const char *checksum, + gboolean store_if_packed, GFileInfo *file_info, GVariant *xattrs, GInputStream *input, GCancellable *cancellable, GError **error) { - return stage_object_impl (self, objtype, + return stage_object_impl (self, objtype, store_if_packed, file_info, xattrs, input, checksum, NULL, cancellable, error); } @@ -1333,7 +1301,7 @@ ostree_repo_stage_object (OstreeRepo *self, gboolean ret = FALSE; GChecksum *actual_checksum = NULL; - if (!stage_object_impl (self, objtype, + if (!stage_object_impl (self, objtype, FALSE, file_info, xattrs, input, expected_checksum, &actual_checksum, cancellable, error)) goto out; @@ -1567,6 +1535,664 @@ ostree_repo_stage_commit (OstreeRepo *self, return ret; } +static gboolean +list_files_in_dir_matching (GFile *dir, + const char *prefix, + const char *suffix, + GPtrArray **out_files, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GError *temp_error = NULL; + GFileEnumerator *enumerator = NULL; + GFileInfo *file_info = NULL; + GPtrArray *ret_files = NULL; + + g_return_val_if_fail (prefix != NULL || suffix != NULL, FALSE); + + ret_files = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); + + enumerator = g_file_enumerate_children (dir, OSTREE_GIO_FAST_QUERYINFO, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + cancellable, + error); + if (!enumerator) + goto out; + + while ((file_info = g_file_enumerator_next_file (enumerator, cancellable, &temp_error)) != NULL) + { + const char *name; + guint32 type; + + name = g_file_info_get_attribute_byte_string (file_info, "standard::name"); + type = g_file_info_get_attribute_uint32 (file_info, "standard::type"); + + if (type != G_FILE_TYPE_REGULAR) + goto loop_next; + + if (prefix) + { + if (!g_str_has_prefix (name, prefix)) + goto loop_next; + } + if (suffix) + { + if (!g_str_has_suffix (name, suffix)) + goto loop_next; + } + + g_ptr_array_add (ret_files, g_file_get_child (dir, name)); + + loop_next: + g_clear_object (&file_info); + } + if (temp_error != NULL) + { + g_propagate_error (error, temp_error); + goto out; + } + if (!g_file_enumerator_close (enumerator, cancellable, error)) + goto out; + + ret = TRUE; + ot_transfer_out_value (out_files, &ret_files); + out: + if (ret_files) + g_ptr_array_unref (ret_files); + g_clear_object (&file_info); + g_clear_object (&enumerator); + return ret; +} + +static gboolean +map_variant_file_check_header_string (GFile *path, + const GVariantType *variant_type, + const char *expected_header, + GVariant **out_variant, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GVariant *ret_variant = NULL; + const char *header; + + if (!ot_util_variant_map (path, variant_type, &ret_variant, error)) + goto out; + + g_variant_get_child (ret_variant, 0, "&s", &header); + + if (strcmp (header, expected_header) != 0) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid variant file '%s', expected header '%s'", + ot_gfile_get_path_cached (path), + expected_header); + goto out; + } + + ret = TRUE; + ot_transfer_out_value (out_variant, &ret_variant); + out: + ot_clear_gvariant (&ret_variant); + return ret; +} + + +static char * +get_checksum_from_pack_name (const char *name) +{ + const char *dash; + const char *dot; + + dash = strchr (name, '-'); + g_assert (dash); + dot = strrchr (name, '.'); + g_assert (dot); + + g_assert_cmpint (dot - (dash + 1), ==, 64); + + return g_strndup (dash + 1, 64); +} + +static gboolean +list_pack_indexes_from_dir (OstreeRepo *self, + GPtrArray **out_indexes, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GPtrArray *index_files = NULL; + GPtrArray *ret_indexes = NULL; + OstreeRepoPrivate *priv = GET_PRIVATE (self); + guint i; + + if (!list_files_in_dir_matching (priv->pack_dir, + "ostpack-", ".index", + &index_files, + cancellable, error)) + goto out; + + ret_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free); + for (i = 0; i < index_files->len; i++) + { + GFile *index_path = index_files->pdata[i]; + const char *basename = ot_gfile_get_basename_cached (index_path); + g_ptr_array_add (ret_indexes, get_checksum_from_pack_name (basename)); + } + + ret = TRUE; + ot_transfer_out_value (out_indexes, &ret_indexes); + out: + if (index_files) + g_ptr_array_unref (index_files); + if (ret_indexes) + g_ptr_array_unref (ret_indexes); + return ret; +} + +static gboolean +list_pack_checksums_from_superindex_file (GFile *superindex_path, + GPtrArray **out_indexes, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GPtrArray *ret_indexes = NULL; + GVariant *superindex_variant = NULL; + GVariantIter *variant_iter = NULL; + const char *magic; + GVariant *checksum; + GVariant *bloom; + + if (!ot_util_variant_map (superindex_path, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT, + &superindex_variant, error)) + goto out; + + g_variant_get (superindex_variant, "(&s@a{sv}a(ayay))", + &magic, NULL, &variant_iter); + + if (strcmp (magic, "OSTv0SUPERPACKINDEX") != 0) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid header in super pack index"); + goto out; + } + + ret_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free); + + while (g_variant_iter_loop (variant_iter, "(@ay@ay)", + &checksum, &bloom)) + g_ptr_array_add (ret_indexes, ostree_checksum_from_bytes (checksum)); + + ret = TRUE; + ot_transfer_out_value (out_indexes, &ret_indexes); + out: + ot_clear_gvariant (&superindex_variant); + if (variant_iter) + g_variant_iter_free (variant_iter); + if (ret_indexes) + g_ptr_array_unref (ret_indexes); + return ret; +} + +gboolean +ostree_repo_list_pack_indexes (OstreeRepo *self, + GPtrArray **out_indexes, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + OstreeRepoPrivate *priv = GET_PRIVATE (self); + GFile *superindex_path = NULL; + GPtrArray *ret_indexes = NULL; + + superindex_path = g_file_get_child (priv->pack_dir, "index"); + + if (g_file_query_exists (superindex_path, cancellable)) + { + if (!list_pack_checksums_from_superindex_file (superindex_path, &ret_indexes, cancellable, error)) + goto out; + } + else + { + ret_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free); + } + + ret = TRUE; + ot_transfer_out_value (out_indexes, &ret_indexes); + out: + g_clear_object (&superindex_path); + if (ret_indexes) + g_ptr_array_unref (ret_indexes); + return ret; +} + +static gboolean +create_index_bloom (OstreeRepo *self, + const char *pack_checksum, + GVariant **out_bloom, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GVariant *ret_bloom; + + /* TODO - define and compute bloom filter */ + + ret_bloom = g_variant_new_fixed_array (G_VARIANT_TYPE ("y"), NULL, 0, 1); + g_variant_ref_sink (ret_bloom); + + ret = TRUE; + ot_transfer_out_value (out_bloom, &ret_bloom); + /* out: */ + ot_clear_gvariant (&ret_bloom); + return ret; +} + +/** + * Regenerate the pack superindex file based on the set of pack + * indexes currently in the filesystem. + */ +gboolean +ostree_repo_regenerate_pack_index (OstreeRepo *self, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + OstreeRepoPrivate *priv = GET_PRIVATE (self); + GFile *index_path = NULL; + GPtrArray *pack_indexes = NULL; + GVariantBuilder *index_content_builder = NULL; + GVariant *index_variant = NULL; + guint i; + + if (!list_pack_indexes_from_dir (self, &pack_indexes, cancellable, error)) + goto out; + + index_path = g_file_get_child (priv->pack_dir, "index"); + + index_content_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(ayay)")); + + for (i = 0; i < pack_indexes->len; i++) + { + const char *pack_checksum = pack_indexes->pdata[i]; + GVariant *bloom; + + if (!create_index_bloom (self, pack_checksum, &bloom, cancellable, error)) + goto out; + + g_variant_builder_add (index_content_builder, + "(@ay@ay)", + ostree_checksum_to_bytes (pack_checksum), + bloom); + g_variant_unref (bloom); + } + + index_variant = g_variant_new ("(s@a{sv}@a(ayay))", + "OSTv0SUPERPACKINDEX", + g_variant_new_array (G_VARIANT_TYPE ("{sv}"), + NULL, 0), + g_variant_builder_end (index_content_builder)); + g_variant_ref_sink (index_variant); + + if (!ot_util_variant_save (index_path, index_variant, + cancellable, error)) + goto out; + + ret = TRUE; + out: + ot_clear_gvariant (&index_variant); + if (index_content_builder) + g_variant_builder_unref (index_content_builder); + g_clear_object (&index_path); + if (pack_indexes) + g_ptr_array_unref (pack_indexes); + return ret; +} + + +static GFile * +get_pack_index_name_from_checksum (GFile *parent, const char *pack_checksum) +{ + return ot_gfile_get_child_strconcat (parent, "ostpack-", pack_checksum, ".index", NULL); +} + +static GFile * +get_pack_data_name_from_checksum (GFile *parent, const char *pack_checksum) +{ + return ot_gfile_get_child_strconcat (parent, "ostpack-", pack_checksum, ".data", NULL); +} + +gboolean +ostree_repo_add_pack_file (OstreeRepo *self, + const char *pack_checksum, + GFile *index_path, + GFile *data_path, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + OstreeRepoPrivate *priv = GET_PRIVATE (self); + GFile *pack_index_path = NULL; + GFile *pack_data_path = NULL; + + if (!ot_gfile_ensure_directory (priv->pack_dir, FALSE, error)) + goto out; + + pack_data_path = get_pack_data_name_from_checksum (priv->pack_dir, pack_checksum); + if (!ot_gfile_rename (data_path, pack_data_path, cancellable, error)) + goto out; + + pack_index_path = get_pack_index_name_from_checksum (priv->pack_dir, pack_checksum); + if (!ot_gfile_rename (index_path, pack_index_path, cancellable, error)) + goto out; + + ret = TRUE; + out: + g_clear_object (&pack_index_path); + g_clear_object (&pack_data_path); + return ret; +} + +static gboolean +ensure_remote_cache_dir (OstreeRepo *self, + const char *remote_name, + GFile **out_cache_dir, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + OstreeRepoPrivate *priv = GET_PRIVATE (self); + GFile *ret_cache_dir = NULL; + + ret_cache_dir = g_file_get_child (priv->remote_cache_dir, remote_name); + + if (!ot_gfile_ensure_directory (ret_cache_dir, FALSE, error)) + goto out; + + ret = TRUE; + ot_transfer_out_value (out_cache_dir, &ret_cache_dir); + out: + g_clear_object (&ret_cache_dir); + return ret; +} + +/** + * Take a pack superindex file @superindex_path, and clean up any + * no-longer-referenced pack files in the lookaside cache for + * @remote_name. The updated index file will also be saved into the + * cache. + * + * Upon successful return, @out_cached_indexes will hold checksum + * strings for indexes which are already in the cache, and + * @out_uncached_indexes will hold strings for those which are not. + */ +gboolean +ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self, + const char *remote_name, + GFile *superindex_path, + GPtrArray **out_cached_indexes, + GPtrArray **out_uncached_indexes, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GVariant *superindex_variant = NULL; + GVariantIter *superindex_contents_iter = NULL; + GFile *cache_path = NULL; + GFile *superindex_cache_path = NULL; + GPtrArray *index_files = NULL; + GHashTable *new_pack_indexes = NULL; + GHashTableIter hash_iter; + gpointer key, value; + GPtrArray *ret_cached_indexes = NULL; + GPtrArray *ret_uncached_indexes = NULL; + guint i; + GVariant *csum_bytes = NULL; + GVariant *bloom = NULL; + char *pack_checksum = NULL; + + if (!ensure_remote_cache_dir (self, remote_name, &cache_path, cancellable, error)) + goto out; + + ret_cached_indexes = g_ptr_array_new_with_free_func (g_free); + ret_uncached_indexes = g_ptr_array_new_with_free_func (g_free); + + if (!ot_util_variant_map (superindex_path, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT, + &superindex_variant, error)) + goto out; + + if (!ostree_validate_structureof_pack_superindex (superindex_variant, error)) + goto out; + + new_pack_indexes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); + g_variant_get_child (superindex_variant, 2, "a(ayay)", + &superindex_contents_iter); + + while (g_variant_iter_loop (superindex_contents_iter, + "(@ay@ay)", &csum_bytes, &bloom)) + { + pack_checksum = ostree_checksum_from_bytes (csum_bytes); + g_hash_table_insert (new_pack_indexes, pack_checksum, pack_checksum); + pack_checksum = NULL; /* transfer ownership */ + } + + if (!list_files_in_dir_matching (cache_path, + "ostpack-", ".index", + &index_files, + cancellable, error)) + goto out; + + for (i = 0; i < index_files->len; i++) + { + GFile *index_file = index_files->pdata[i]; + + g_free (pack_checksum); + pack_checksum = get_checksum_from_pack_name (ot_gfile_get_basename_cached (index_file)); + + if (!g_hash_table_lookup (new_pack_indexes, pack_checksum)) + { + if (!ot_gfile_unlink (index_file, cancellable, error)) + goto out; + } + + g_ptr_array_add (ret_cached_indexes, pack_checksum); + pack_checksum = NULL; /* transfer ownership */ + } + + g_hash_table_iter_init (&hash_iter, new_pack_indexes); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + const char *cur_pack_checksum = key; + gboolean found = FALSE; + + for (i = 0; i < ret_cached_indexes->len; i++) + { + if (strcmp (cur_pack_checksum, ret_cached_indexes->pdata[i]) == 0) + { + found = TRUE; + break; + } + } + + if (!found) + g_ptr_array_add (ret_uncached_indexes, g_strdup (cur_pack_checksum)); + } + + superindex_cache_path = g_file_get_child (cache_path, "index"); + if (!ot_util_variant_save (superindex_cache_path, superindex_variant, cancellable, error)) + goto out; + + ret = TRUE; + ot_transfer_out_value (out_cached_indexes, &ret_cached_indexes); + ot_transfer_out_value (out_uncached_indexes, &ret_uncached_indexes); + out: + g_free (pack_checksum); + g_clear_object (&cache_path); + g_clear_object (&superindex_cache_path); + if (superindex_contents_iter) + g_variant_iter_free (superindex_contents_iter); + ot_clear_ptrarray (&ret_cached_indexes); + ot_clear_ptrarray (&ret_uncached_indexes); + ot_clear_hashtable (&new_pack_indexes); + ot_clear_ptrarray (&index_files); + return ret; +} + +/** + * Load the index for pack @pack_checksum from cache directory for + * @remote_name. + */ +gboolean +ostree_repo_map_cached_remote_pack_index (OstreeRepo *self, + const char *remote_name, + const char *pack_checksum, + GVariant **out_variant, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GVariant *ret_variant = NULL; + GFile *cache_dir = NULL; + GFile *cached_pack_path = NULL; + + if (!ensure_remote_cache_dir (self, remote_name, &cache_dir, + cancellable, error)) + goto out; + + cached_pack_path = get_pack_index_name_from_checksum (cache_dir, pack_checksum); + if (!ot_util_variant_map (cached_pack_path, OSTREE_PACK_INDEX_VARIANT_FORMAT, + &ret_variant, error)) + goto out; + + ret = TRUE; + ot_transfer_out_value (out_variant, &ret_variant); + out: + g_clear_object (&cache_dir); + g_clear_object (&cached_pack_path); + ot_clear_gvariant (&ret_variant); + return ret; +} + +/** + * The variable @cached_path should refer to a file containing a pack + * index. It will be validated and added to the cache directory for + * @remote_name. + */ +gboolean +ostree_repo_add_cached_remote_pack_index (OstreeRepo *self, + const char *remote_name, + const char *pack_checksum, + GFile *cached_path, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GFile *cachedir = NULL; + GFile *target_path = NULL; + GVariant *input_index_variant = NULL; + GVariant *output_index_variant = NULL; + + if (!map_variant_file_check_header_string (cached_path, + OSTREE_PACK_INDEX_VARIANT_FORMAT, + "OSTv0PACKINDEX", + &input_index_variant, + cancellable, error)) + goto out; + + if (!ostree_validate_structureof_pack_index (input_index_variant, error)) + goto out; + + output_index_variant = g_variant_get_normal_form (input_index_variant); + + if (!ensure_remote_cache_dir (self, remote_name, &cachedir, cancellable, error)) + goto out; + + target_path = get_pack_index_name_from_checksum (cachedir, pack_checksum); + if (!ot_util_variant_save (target_path, output_index_variant, cancellable, error)) + goto out; + + ret = TRUE; + out: + g_clear_object (&cachedir); + g_clear_object (&target_path); + ot_clear_gvariant (&input_index_variant); + ot_clear_gvariant (&output_index_variant); + return ret; +} + +/** + * Check for availability of the pack index pointing to @pack_checksum + * in the lookaside cache for @remote_name. If not found, then the + * output parameter @out_cached_path will be %NULL. + */ +gboolean +ostree_repo_get_cached_remote_pack_data (OstreeRepo *self, + const char *remote_name, + const char *pack_checksum, + GFile **out_cached_path, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GFile *cache_dir = NULL; + GFile *cached_pack_path = NULL; + GFile *ret_cached_path = NULL; + + if (!ensure_remote_cache_dir (self, remote_name, &cache_dir, + cancellable, error)) + goto out; + + cached_pack_path = get_pack_data_name_from_checksum (cache_dir, pack_checksum); + if (g_file_query_exists (cached_pack_path, cancellable)) + { + ret_cached_path = cached_pack_path; + cached_pack_path = NULL; + } + + ret = TRUE; + ot_transfer_out_value (out_cached_path, &ret_cached_path); + out: + g_clear_object (&cache_dir); + g_clear_object (&cached_pack_path); + return ret; +} + +/** + * Add file @cached_path into the cache for given @remote_name. + * + * + * This unlinks @cached_path. + * + */ +gboolean +ostree_repo_take_cached_remote_pack_data (OstreeRepo *self, + const char *remote_name, + const char *pack_checksum, + GFile *cached_path, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GFile *cachedir = NULL; + GFile *target_path = NULL; + + if (!ensure_remote_cache_dir (self, remote_name, &cachedir, cancellable, error)) + goto out; + + target_path = get_pack_data_name_from_checksum (cachedir, pack_checksum); + + if (!ot_gfile_rename (cached_path, target_path, cancellable, error)) + goto out; + + ret = TRUE; + out: + g_clear_object (&cachedir); + g_clear_object (&target_path); + return ret; +} + static GVariant * create_tree_variant_from_hashes (GHashTable *file_checksums, GHashTable *dir_contents_checksums, @@ -1819,7 +2445,7 @@ stage_directory_to_mtree_internal (OstreeRepo *self, goto out; } - if (!stage_object_impl (self, OSTREE_OBJECT_TYPE_RAW_FILE, + if (!stage_object_impl (self, OSTREE_OBJECT_TYPE_RAW_FILE, FALSE, modified_info, xattrs, file_input, NULL, &child_file_checksum, cancellable, error)) goto out; @@ -2021,7 +2647,7 @@ import_libarchive_entry_file (OstreeRepo *self, if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) archive_stream = ostree_libarchive_input_stream_new (a); - if (!stage_object_impl (self, OSTREE_OBJECT_TYPE_RAW_FILE, + if (!stage_object_impl (self, OSTREE_OBJECT_TYPE_RAW_FILE, FALSE, file_info, NULL, archive_stream, NULL, &ret_checksum, cancellable, error)) @@ -2294,43 +2920,42 @@ ostree_repo_commit_modifier_unref (OstreeRepoCommitModifier *modifier) return; } - static gboolean -iter_object_dir (OstreeRepo *self, - GFile *dir, - OstreeRepoObjectIter callback, - gpointer user_data, - GError **error) +list_loose_object_dir (OstreeRepo *self, + GFile *dir, + GHashTable *inout_objects, + GCancellable *cancellable, + GError **error) { gboolean ret = FALSE; GError *temp_error = NULL; GFileEnumerator *enumerator = NULL; GFileInfo *file_info = NULL; const char *dirname = NULL; + char *dot = NULL; + GString *checksum = NULL; dirname = ot_gfile_get_basename_cached (dir); - enumerator = g_file_enumerate_children (dir, OSTREE_GIO_FAST_QUERYINFO, + /* We're only querying name */ + enumerator = g_file_enumerate_children (dir, "standard::name,standard::type", G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - NULL, + cancellable, error); if (!enumerator) goto out; - while ((file_info = g_file_enumerator_next_file (enumerator, NULL, &temp_error)) != NULL) + while ((file_info = g_file_enumerator_next_file (enumerator, cancellable, &temp_error)) != NULL) { const char *name; guint32 type; - char *dot = NULL; - GFile *child = NULL; - GString *checksum = NULL; OstreeObjectType objtype; name = g_file_info_get_attribute_byte_string (file_info, "standard::name"); type = g_file_info_get_attribute_uint32 (file_info, "standard::type"); if (type == G_FILE_TYPE_DIRECTORY) - goto loop_out; + goto loop_next; if (g_str_has_suffix (name, ".file")) objtype = OSTREE_OBJECT_TYPE_RAW_FILE; @@ -2345,25 +2970,29 @@ iter_object_dir (OstreeRepo *self, else if (g_str_has_suffix (name, ".commit")) objtype = OSTREE_OBJECT_TYPE_COMMIT; else - goto loop_out; + goto loop_next; dot = strrchr (name, '.'); g_assert (dot); - if ((dot - name) != 62) - goto loop_out; - - checksum = g_string_new (dirname); - g_string_append_len (checksum, name, 62); - - child = g_file_get_child (dir, name); - callback (self, checksum->str, objtype, child, file_info, user_data); - - loop_out: - if (checksum) - g_string_free (checksum, TRUE); + if ((dot - name) == 62) + { + GVariant *key, *value; + + if (checksum) + g_string_free (checksum, TRUE); + checksum = g_string_new (dirname); + g_string_append_len (checksum, name, 62); + + key = ostree_object_name_serialize (checksum->str, objtype); + value = g_variant_new ("(b@as)", + TRUE, g_variant_new_strv (NULL, 0)); + /* transfer ownership */ + g_hash_table_replace (inout_objects, g_variant_ref_sink (key), + g_variant_ref_sink (value)); + } + loop_next: g_clear_object (&file_info); - g_clear_object (&child); } if (temp_error != NULL) { @@ -2376,32 +3005,33 @@ iter_object_dir (OstreeRepo *self, ret = TRUE; out: g_clear_object (&file_info); + g_clear_object (&enumerator); + if (checksum) + g_string_free (checksum, TRUE); return ret; } -gboolean -ostree_repo_iter_objects (OstreeRepo *self, - OstreeRepoObjectIter callback, - gpointer user_data, - GError **error) +static gboolean +list_loose_objects (OstreeRepo *self, + GHashTable *inout_objects, + GCancellable *cancellable, + GError **error) { + gboolean ret = FALSE; OstreeRepoPrivate *priv = GET_PRIVATE (self); GFileEnumerator *enumerator = NULL; - gboolean ret = FALSE; GFileInfo *file_info = NULL; GError *temp_error = NULL; - - g_return_val_if_fail (error == NULL || *error == NULL, FALSE); - g_return_val_if_fail (priv->inited, FALSE); + GFile *objdir = NULL; enumerator = g_file_enumerate_children (priv->objects_dir, OSTREE_GIO_FAST_QUERYINFO, G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - NULL, + cancellable, error); if (!enumerator) goto out; - while ((file_info = g_file_enumerator_next_file (enumerator, NULL, &temp_error)) != NULL) + while ((file_info = g_file_enumerator_next_file (enumerator, cancellable, &temp_error)) != NULL) { const char *name; guint32 type; @@ -2411,31 +3041,608 @@ ostree_repo_iter_objects (OstreeRepo *self, if (strlen (name) == 2 && type == G_FILE_TYPE_DIRECTORY) { - GFile *objdir = g_file_get_child (priv->objects_dir, name); - if (!iter_object_dir (self, objdir, callback, user_data, error)) - { - g_object_unref (objdir); - goto out; - } - g_object_unref (objdir); + g_clear_object (&objdir); + objdir = g_file_get_child (priv->objects_dir, name); + if (!list_loose_object_dir (self, objdir, inout_objects, cancellable, error)) + goto out; } - g_object_unref (file_info); + g_clear_object (&file_info); } if (file_info == NULL && temp_error != NULL) { g_propagate_error (error, temp_error); goto out; } - if (!g_file_enumerator_close (enumerator, NULL, error)) + if (!g_file_enumerator_close (enumerator, cancellable, error)) goto out; ret = TRUE; out: + g_clear_object (&objdir); g_clear_object (&file_info); g_clear_object (&enumerator); return ret; } +GFile * +ostree_repo_get_pack_index_path (OstreeRepo *self, + const char *checksum) +{ + char *name; + GFile *ret; + + name = g_strconcat ("ostpack-", checksum, ".index", NULL); + ret = g_file_get_child (GET_PRIVATE (self)->pack_dir, name); + g_free (name); + + return ret; +} + +GFile * +ostree_repo_get_pack_data_path (OstreeRepo *self, + const char *checksum) +{ + char *name; + GFile *ret; + + name = g_strconcat ("ostpack-", checksum, ".data", NULL); + ret = g_file_get_child (GET_PRIVATE (self)->pack_dir, name); + g_free (name); + + return ret; +} + +gboolean +ostree_repo_load_pack_index (OstreeRepo *self, + const char *sha256, + GVariant **out_variant, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + OstreeRepoPrivate *priv = GET_PRIVATE (self); + GVariant *ret_variant = NULL; + GFile *path = NULL; + + ret_variant = g_hash_table_lookup (priv->pack_index_mappings, sha256); + if (ret_variant) + { + g_variant_ref (ret_variant); + } + else + { + path = ostree_repo_get_pack_index_path (self, sha256); + if (!map_variant_file_check_header_string (path, + OSTREE_PACK_INDEX_VARIANT_FORMAT, + "OSTv0PACKINDEX", + &ret_variant, + cancellable, error)) + goto out; + g_hash_table_insert (priv->pack_index_mappings, g_strdup (sha256), + g_variant_ref (ret_variant)); + } + + ret = TRUE; + ot_transfer_out_value (out_variant, &ret_variant); + out: + g_clear_object (&path); + ot_clear_gvariant (&ret_variant); + return ret; +} + +/** + * ostree_repo_map_pack_file: + * @self: + * @sha256: Checksum of pack file + * @out_data: (out): Pointer to pack file data + * @cancellable: + * @error: + * + * Ensure that the given pack file is mapped into + * memory. + */ +gboolean +ostree_repo_map_pack_file (OstreeRepo *self, + const char *sha256, + guchar **out_data, + guint64 *out_len, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + OstreeRepoPrivate *priv = GET_PRIVATE (self); + GMappedFile *map; + gpointer ret_data; + guint64 ret_len; + GFile *path = NULL; + + map = g_hash_table_lookup (priv->pack_data_mappings, sha256); + if (map == NULL) + { + path = ostree_repo_get_pack_data_path (self, sha256); + + map = g_mapped_file_new (ot_gfile_get_path_cached (path), FALSE, error); + if (!map) + goto out; + + g_hash_table_insert (priv->pack_data_mappings, g_strdup (sha256), map); + ret_data = g_mapped_file_get_contents (map); + } + + ret_data = g_mapped_file_get_contents (map); + ret_len = (guint64)g_mapped_file_get_length (map); + + ret = TRUE; + if (out_data) + *out_data = ret_data; + if (out_len) + *out_len = ret_len; + out: + g_clear_object (&path); + return ret; +} + + +gboolean +ostree_repo_load_file (OstreeRepo *self, + const char *checksum, + GInputStream **out_input, + GFileInfo **out_file_info, + GVariant **out_xattrs, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GVariant *archive_meta = NULL; + GFile *content_loose_path = NULL; + GFileInfo *content_loose_info = NULL; + char *content_pack_checksum = NULL; + guint64 content_pack_offset; + guchar *content_pack_data; + guint64 content_pack_len; + GVariant *packed_object = NULL; + GInputStream *ret_input = NULL; + GFileInfo *ret_file_info = NULL; + GVariant *ret_xattrs = NULL; + + if (ostree_repo_get_mode (self) == OSTREE_REPO_MODE_ARCHIVE) + { + /* First, read the metadata */ + if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, checksum, + &archive_meta, error)) + goto out; + if (!ostree_parse_archived_file_meta (archive_meta, + &ret_file_info, + &ret_xattrs, + error)) + goto out; + + /* Blah, right now we need to look up the content too to get the file size */ + if (!ostree_repo_find_object (self, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, + checksum, &content_loose_path, NULL, + &content_pack_checksum, &content_pack_offset, + cancellable, error)) + goto out; + + if (content_loose_path) + { + content_loose_info = g_file_query_info (content_loose_path, OSTREE_GIO_FAST_QUERYINFO, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, cancellable, error); + if (!content_loose_info) + goto out; + + g_file_info_set_attribute_uint64 (ret_file_info, + "standard::size", + g_file_info_get_attribute_uint64 (content_loose_info, "standard::size")); + } + /* fixme - don't have file size for packed =/ */ + + /* Now, look for the content */ + if (g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_REGULAR + && out_input) + { + if (content_pack_checksum != NULL) + { + if (!ostree_repo_map_pack_file (self, content_pack_checksum, + &content_pack_data, &content_pack_len, + cancellable, error)) + goto out; + if (!ostree_read_pack_entry_raw (content_pack_data, content_pack_len, + content_pack_offset, TRUE, + &packed_object, cancellable, error)) + goto out; + ret_input = ostree_read_pack_entry_as_stream (packed_object); + } + else if (content_loose_path != NULL) + { + ret_input = (GInputStream*)g_file_read (content_loose_path, cancellable, error); + if (!ret_input) + goto out; + } + else + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "Couldn't find object '%s'", checksum); + goto out; + } + } + } + else + { + content_loose_path = ostree_repo_get_object_path (self, checksum, OSTREE_OBJECT_TYPE_RAW_FILE); + ret_file_info = g_file_query_info (content_loose_path, OSTREE_GIO_FAST_QUERYINFO, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, cancellable, error); + if (!ret_file_info) + goto out; + if (out_xattrs) + { + ret_xattrs = ostree_get_xattrs_for_file (content_loose_path, error); + if (!ret_xattrs) + goto out; + } + } + + ret = TRUE; + ot_transfer_out_value (out_input, &ret_input); + ot_transfer_out_value (out_file_info, &ret_file_info); + ot_transfer_out_value (out_xattrs, &ret_xattrs); + out: + g_free (content_pack_checksum); + g_clear_object (&ret_input); + g_clear_object (&content_loose_info); + g_clear_object (&ret_file_info); + ot_clear_gvariant (&ret_xattrs); + ot_clear_gvariant (&archive_meta); + ot_clear_gvariant (&packed_object); + return ret; +} + +static gboolean +list_objects_in_index (OstreeRepo *self, + const char *pack_checksum, + GHashTable *inout_objects, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GFile *index_path = NULL; + GVariant *index_variant = NULL; + GVariant *contents; + GVariantIter content_iter; + GVariant *csum_bytes; + char *checksum = NULL; + guint32 objtype_u32; + guint64 offset; + + index_path = ostree_repo_get_pack_index_path (self, pack_checksum); + + if (!ostree_repo_load_pack_index (self, pack_checksum, &index_variant, cancellable, error)) + goto out; + + contents = g_variant_get_child_value (index_variant, 2); + g_variant_iter_init (&content_iter, contents); + + while (g_variant_iter_loop (&content_iter, "(u@ayt)", &objtype_u32, &csum_bytes, &offset)) + { + GVariant *obj_key; + GVariant *objdata; + OstreeObjectType objtype; + GVariantBuilder pack_contents_builder; + gboolean is_loose; + + objtype = (OstreeObjectType) GUINT32_FROM_BE (objtype_u32); + offset = GUINT64_FROM_BE (offset); + + g_variant_builder_init (&pack_contents_builder, + G_VARIANT_TYPE_STRING_ARRAY); + + g_free (checksum); + checksum = ostree_checksum_from_bytes (csum_bytes); + obj_key = ostree_object_name_serialize (checksum, objtype); + ot_util_variant_take_ref (obj_key); + + objdata = g_hash_table_lookup (inout_objects, obj_key); + if (!objdata) + { + is_loose = FALSE; + } + else + { + GVariantIter *current_packs_iter; + const char *current_pack_checksum; + + g_variant_get (objdata, "(bas)", &is_loose, ¤t_packs_iter); + + while (g_variant_iter_loop (current_packs_iter, "&s", ¤t_pack_checksum)) + { + g_variant_builder_add (&pack_contents_builder, "s", current_pack_checksum); + } + g_variant_iter_free (current_packs_iter); + } + g_variant_builder_add (&pack_contents_builder, "s", pack_checksum); + objdata = g_variant_new ("(b@as)", is_loose, + g_variant_builder_end (&pack_contents_builder)); + g_hash_table_replace (inout_objects, + obj_key, + g_variant_ref (objdata)); + } + + ret = TRUE; + out: + g_free (checksum); + g_clear_object (&index_path); + ot_clear_gvariant (&index_variant); + ot_clear_gvariant (&contents); + return ret; +} + +static gboolean +list_packed_objects (OstreeRepo *self, + GHashTable *inout_objects, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GPtrArray *index_checksums = NULL; + guint i; + + if (!ostree_repo_list_pack_indexes (self, &index_checksums, cancellable, error)) + goto out; + + for (i = 0; i < index_checksums->len; i++) + { + const char *checksum = index_checksums->pdata[i]; + + if (!list_objects_in_index (self, checksum, inout_objects, cancellable, error)) + goto out; + } + + ret = TRUE; + out: + if (index_checksums) + g_ptr_array_unref (index_checksums); + return ret; +} + +static gboolean +find_object_in_packs (OstreeRepo *self, + const char *checksum, + OstreeObjectType objtype, + char **out_pack_checksum, + guint64 *out_pack_offset, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GPtrArray *index_checksums = NULL; + char *ret_pack_checksum = NULL; + guint64 ret_pack_offset; + GFile *index_path = NULL; + GVariant *csum_bytes = NULL; + GVariant *index_variant = NULL; + guint i; + + csum_bytes = ostree_checksum_to_bytes (checksum); + + if (!ostree_repo_list_pack_indexes (self, &index_checksums, cancellable, error)) + goto out; + + for (i = 0; i < index_checksums->len; i++) + { + const char *pack_checksum = index_checksums->pdata[i]; + guint64 offset; + + g_clear_object (&index_path); + index_path = ostree_repo_get_pack_index_path (self, pack_checksum); + + ot_clear_gvariant (&index_variant); + if (!ostree_repo_load_pack_index (self, pack_checksum, &index_variant, cancellable, error)) + goto out; + + if (!ostree_pack_index_search (index_variant, csum_bytes, objtype, &offset)) + continue; + + ret_pack_checksum = g_strdup (pack_checksum); + ret_pack_offset = offset; + break; + } + + ret = TRUE; + ot_transfer_out_value (out_pack_checksum, &ret_pack_checksum); + if (out_pack_offset) + *out_pack_offset = ret_pack_offset; + out: + g_free (ret_pack_checksum); + if (index_checksums) + g_ptr_array_unref (index_checksums); + g_clear_object (&index_path); + ot_clear_gvariant (&index_variant); + ot_clear_gvariant (&csum_bytes); + return ret; +} + +gboolean +ostree_repo_find_object (OstreeRepo *self, + OstreeObjectType objtype, + const char *checksum, + GFile **out_stored_path, + GFile **out_pending_path, + char **out_pack_checksum, + guint64 *out_pack_offset, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GFile *object_path = NULL; + GFile *ret_stored_path = NULL; + GFile *ret_pending_path = NULL; + char *ret_pack_checksum = NULL; + guint64 ret_pack_offset = 0; + struct stat stbuf; + + object_path = ostree_repo_get_object_path (self, checksum, objtype); + + if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0) + { + ret_stored_path = object_path; + object_path = NULL; + } + else + { + g_clear_object (&object_path); + if (out_pending_path) + { + object_path = get_pending_object_path (self, checksum, objtype); + if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0) + { + ret_pending_path = object_path; + object_path = NULL; + } + } + } + + if (out_pack_checksum) + { + if (!find_object_in_packs (self, checksum, objtype, + &ret_pack_checksum, &ret_pack_offset, + cancellable, error)) + goto out; + } + + ret = TRUE; + ot_transfer_out_value (out_stored_path, &ret_stored_path); + ot_transfer_out_value (out_pending_path, &ret_pending_path); + ot_transfer_out_value (out_pack_checksum, &ret_pack_checksum); + if (out_pack_offset) + *out_pack_offset = ret_pack_offset; +out: + g_clear_object (&object_path); + g_clear_object (&ret_stored_path); + g_clear_object (&ret_pending_path); + g_free (ret_pack_checksum); + return ret; +} + +gboolean +ostree_repo_load_variant (OstreeRepo *self, + OstreeObjectType objtype, + const char *sha256, + GVariant **out_variant, + GError **error) +{ + gboolean ret = FALSE; + GFile *object_path = NULL; + GFile *pending_path = NULL; + GVariant *packed_object = NULL; + GVariant *ret_variant = NULL; + char *pack_checksum = NULL; + guchar *pack_data; + guint64 pack_len; + guint64 object_offset; + GCancellable *cancellable = NULL; + + g_return_val_if_fail (OSTREE_OBJECT_TYPE_IS_META (objtype), FALSE); + + if (!ostree_repo_find_object (self, objtype, sha256, &object_path, &pending_path, + &pack_checksum, &object_offset, + cancellable, error)) + goto out; + + /* Prefer loose metadata for now */ + if (object_path != NULL || pending_path != NULL) + { + if (!ostree_map_metadata_file (object_path ? object_path : pending_path, + objtype, &ret_variant, error)) + goto out; + } + else if (pack_checksum != NULL) + { + if (!ostree_repo_map_pack_file (self, pack_checksum, &pack_data, &pack_len, + cancellable, error)) + goto out; + + if (!ostree_read_pack_entry_raw (pack_data, pack_len, object_offset, + TRUE, &packed_object, cancellable, error)) + goto out; + + if (!ostree_read_pack_entry_variant (packed_object, objtype, TRUE, + &ret_variant, cancellable, error)) + goto out; + } + else + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "No such metadata object %s.%s", + sha256, ostree_object_type_to_string (objtype)); + goto out; + } + + ret = TRUE; + ot_transfer_out_value (out_variant, &ret_variant); + out: + g_clear_object (&object_path); + g_free (pack_checksum); + ot_clear_gvariant (&ret_variant); + ot_clear_gvariant (&packed_object); + return ret; +} + +/** + * ostree_repo_list_objects: + * @self: + * @flags: + * @out_objects: (out): Map of serialized object name to variant data + * @cancellable: + * @error: + * + * This function synchronously enumerates all objects in the + * repository, returning data in @out_objects. @out_objects + * maps from keys returned by ostree_object_name_serialize() + * to #GVariant values of type %OSTREE_REPO_LIST_OBJECTS_VARIANT_TYPE. + * + * Returns: %TRUE on success, %FALSE on error, and @error will be set + */ +gboolean +ostree_repo_list_objects (OstreeRepo *self, + OstreeRepoListObjectsFlags flags, + GHashTable **out_objects, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + OstreeRepoPrivate *priv = GET_PRIVATE (self); + GHashTable *ret_objects = NULL; + + g_return_val_if_fail (error == NULL || *error == NULL, FALSE); + g_return_val_if_fail (priv->inited, FALSE); + + ret_objects = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, + (GDestroyNotify) g_variant_unref, + (GDestroyNotify) g_variant_unref); + + if (flags & OSTREE_REPO_LIST_OBJECTS_ALL) + flags |= (OSTREE_REPO_LIST_OBJECTS_LOOSE | OSTREE_REPO_LIST_OBJECTS_PACKED); + + if (flags & OSTREE_REPO_LIST_OBJECTS_LOOSE) + { + if (!list_loose_objects (self, ret_objects, cancellable, error)) + goto out; + } + + if (flags & OSTREE_REPO_LIST_OBJECTS_PACKED) + { + if (!list_packed_objects (self, ret_objects, cancellable, error)) + goto out; + } + + ret = TRUE; + ot_transfer_out_value (out_objects, &ret_objects); + out: + if (ret_objects) + g_hash_table_unref (ret_objects); + return ret; +} + static gboolean checkout_file_from_input (GFile *file, OstreeRepoCheckoutMode mode, @@ -2574,6 +3781,61 @@ checkout_file_hardlink (OstreeRepo *self, return ret; } +static gboolean +checkout_one_file (OstreeRepo *self, + OstreeRepoCheckoutMode mode, + OstreeRepoCheckoutOverwriteMode overwrite_mode, + OstreeRepoFile *src, + GFileInfo *file_info, + GFile *destination, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + OstreeRepoPrivate *priv = GET_PRIVATE (self); + GFile *possible_loose_path = NULL; + GInputStream *input = NULL; + GVariant *xattrs = NULL; + const char *checksum; + struct stat stbuf; + + checksum = ostree_repo_file_get_checksum ((OstreeRepoFile*)src); + + /* First check for a loose object */ + if (priv->mode == OSTREE_REPO_MODE_ARCHIVE && mode == OSTREE_REPO_CHECKOUT_MODE_USER) + { + possible_loose_path = ostree_repo_get_object_path (self, checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT); + } + else if (priv->mode == OSTREE_REPO_MODE_BARE && mode == OSTREE_REPO_CHECKOUT_MODE_NONE) + { + possible_loose_path = ostree_repo_get_object_path (self, checksum, OSTREE_OBJECT_TYPE_RAW_FILE); + } + + if (possible_loose_path && lstat (ot_gfile_get_path_cached (possible_loose_path), &stbuf) >= 0) + { + /* If we found one, we can just hardlink */ + if (!checkout_file_hardlink (self, mode, overwrite_mode, possible_loose_path, destination, + cancellable, error) < 0) + goto out; + } + else + { + if (!ostree_repo_load_file (self, checksum, &input, NULL, &xattrs, cancellable, error)) + goto out; + + if (!checkout_file_from_input (destination, mode, overwrite_mode, file_info, xattrs, + input, cancellable, error)) + goto out; + } + + ret = TRUE; + out: + g_clear_object (&possible_loose_path); + g_clear_object (&input); + ot_clear_gvariant (&xattrs); + return ret; +} + gboolean ostree_repo_checkout_tree (OstreeRepo *self, OstreeRepoCheckoutMode mode, @@ -2584,18 +3846,13 @@ ostree_repo_checkout_tree (OstreeRepo *self, GCancellable *cancellable, GError **error) { - OstreeRepoPrivate *priv = GET_PRIVATE (self); gboolean ret = FALSE; GError *temp_error = NULL; - GVariant *archive_metadata = NULL; GFileInfo *file_info = NULL; GVariant *xattrs = NULL; GFileEnumerator *dir_enum = NULL; GFile *src_child = NULL; GFile *dest_path = NULL; - GFile *object_path = NULL; - GFile *content_object_path = NULL; - GInputStream *content_input = NULL; if (!ostree_repo_file_get_xattrs (source, &xattrs, NULL, error)) goto out; @@ -2637,49 +3894,10 @@ ostree_repo_checkout_tree (OstreeRepo *self, } else { - const char *checksum = ostree_repo_file_get_checksum ((OstreeRepoFile*)src_child); - - if (priv->mode == OSTREE_REPO_MODE_ARCHIVE && mode == OSTREE_REPO_CHECKOUT_MODE_USER) - { - g_clear_object (&object_path); - object_path = ostree_repo_get_object_path (self, checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT); - - if (!checkout_file_hardlink (self, mode, overwrite_mode, object_path, dest_path, cancellable, error) < 0) - goto out; - } - else if (priv->mode == OSTREE_REPO_MODE_ARCHIVE) - { - ot_clear_gvariant (&archive_metadata); - if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, checksum, &archive_metadata, error)) - goto out; - - ot_clear_gvariant (&xattrs); - if (!ostree_parse_archived_file_meta (archive_metadata, NULL, &xattrs, error)) - goto out; - - g_clear_object (&content_object_path); - content_object_path = ostree_repo_get_object_path (self, checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT); - - g_clear_object (&content_input); - if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) - { - content_input = (GInputStream*)g_file_read (content_object_path, cancellable, error); - if (!content_input) - goto out; - } - - if (!checkout_file_from_input (dest_path, mode, overwrite_mode, file_info, xattrs, - content_input, cancellable, error)) - goto out; - } - else - { - g_clear_object (&object_path); - object_path = ostree_repo_get_object_path (self, checksum, OSTREE_OBJECT_TYPE_RAW_FILE); - - if (!checkout_file_hardlink (self, mode, overwrite_mode, object_path, dest_path, cancellable, error) < 0) - goto out; - } + if (!checkout_one_file (self, mode, overwrite_mode, + (OstreeRepoFile*)src_child, file_info, + dest_path, cancellable, error)) + goto out; } g_clear_object (&file_info); @@ -2695,15 +3913,12 @@ ostree_repo_checkout_tree (OstreeRepo *self, g_clear_object (&dir_enum); g_clear_object (&file_info); ot_clear_gvariant (&xattrs); - ot_clear_gvariant (&archive_metadata); g_clear_object (&src_child); - g_clear_object (&object_path); - g_clear_object (&content_object_path); - g_clear_object (&content_input); g_clear_object (&dest_path); g_free (dest_path); return ret; } + gboolean ostree_repo_read_commit (OstreeRepo *self, const char *rev, diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 25ef09ee..55c918e2 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -97,6 +97,8 @@ gboolean ostree_repo_find_object (OstreeRepo *self, const char *checksum, GFile **out_stored_path, GFile **out_pending_path, + char **out_pack_checksum, + guint64 *out_pack_offset, GCancellable *cancellable, GError **error); @@ -112,6 +114,7 @@ gboolean ostree_repo_stage_object (OstreeRepo *self, gboolean ostree_repo_stage_object_trusted (OstreeRepo *self, OstreeObjectType objtype, const char *checksum, + gboolean store_if_packed, GFileInfo *file_info, GVariant *xattrs, GInputStream *content, @@ -141,6 +144,33 @@ gboolean ostree_repo_load_variant (OstreeRepo *self, GVariant **out_variant, GError **error); +gboolean ostree_repo_load_pack_index (OstreeRepo *self, + const char *sha256, + GVariant **out_variant, + GCancellable *cancellable, + GError **error); + +gboolean ostree_repo_load_pack_data (OstreeRepo *self, + const char *sha256, + guchar **out_data, + GCancellable *cancellable, + GError **error); + +gboolean ostree_repo_map_pack_file (OstreeRepo *self, + const char *sha256, + guchar **out_data, + guint64 *out_len, + GCancellable *cancellable, + GError **error); + +gboolean ostree_repo_load_file (OstreeRepo *self, + const char *entry_sha256, + GInputStream **out_input, + GFileInfo **out_file_info, + GVariant **out_xattrs, + GCancellable *cancellable, + GError **error); + typedef enum { OSTREE_REPO_COMMIT_FILTER_ALLOW, OSTREE_REPO_COMMIT_FILTER_SKIP @@ -200,6 +230,53 @@ gboolean ostree_repo_stage_commit (OstreeRepo *self, GCancellable *cancellable, GError **error); +gboolean ostree_repo_regenerate_pack_index (OstreeRepo *self, + GCancellable *cancellable, + GError **error); + +gboolean ostree_repo_add_pack_file (OstreeRepo *self, + const char *checksum, + GFile *pack_index_path, + GFile *pack_data_path, + GCancellable *cancellable, + GError **error); + +gboolean ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self, + const char *remote_name, + GFile *superindex_path, + GPtrArray **out_cached_indexes, + GPtrArray **out_uncached_indexes, + GCancellable *cancellable, + GError **error); + +gboolean ostree_repo_map_cached_remote_pack_index (OstreeRepo *self, + const char *remote_name, + const char *pack_checksum, + GVariant **out_variant, + GCancellable *cancellable, + GError **error); + +gboolean ostree_repo_add_cached_remote_pack_index (OstreeRepo *self, + const char *remote_name, + const char *pack_checksum, + GFile *cached_path, + GCancellable *cancellable, + GError **error); + +gboolean ostree_repo_get_cached_remote_pack_data (OstreeRepo *self, + const char *remote_name, + const char *pack_checksum, + GFile **out_cached_path, + GCancellable *cancellable, + GError **error); + +gboolean ostree_repo_take_cached_remote_pack_data (OstreeRepo *self, + const char *remote_name, + const char *pack_checksum, + GFile *cached_path, + GCancellable *cancellable, + GError **error); + typedef enum { OSTREE_REPO_CHECKOUT_MODE_NONE = 0, OSTREE_REPO_CHECKOUT_MODE_USER = 1 @@ -226,17 +303,36 @@ gboolean ostree_repo_read_commit (OstreeRepo *self, GCancellable *cancellable, GError **error); -typedef void (*OstreeRepoObjectIter) (OstreeRepo *self, - const char *checksum, - OstreeObjectType type, - GFile *path, - GFileInfo *fileinfo, - gpointer user_data); +typedef enum { + OSTREE_REPO_LIST_OBJECTS_LOOSE = (1 << 0), + OSTREE_REPO_LIST_OBJECTS_PACKED = (1 << 1), + OSTREE_REPO_LIST_OBJECTS_ALL = (1 << 2) +} OstreeRepoListObjectsFlags; -gboolean ostree_repo_iter_objects (OstreeRepo *self, - OstreeRepoObjectIter callback, - gpointer user_data, - GError **error); +/** + * OSTREE_REPO_LIST_OBJECTS_VARIANT_TYPE: + * + * b - %TRUE if object is available "loose" + * as - List of pack file checksums in which this object appears + */ +#define OSTREE_REPO_LIST_OBJECTS_VARIANT_TYPE (G_VARIANT_TYPE ("(bas)") + +gboolean ostree_repo_list_objects (OstreeRepo *self, + OstreeRepoListObjectsFlags flags, + GHashTable **out_objects, + GCancellable *cancellable, + GError **error); + +gboolean ostree_repo_list_pack_indexes (OstreeRepo *self, + GPtrArray **out_indexes, + GCancellable *cancellable, + GError **error); + +GFile * ostree_repo_get_pack_index_path (OstreeRepo *self, + const char *checksum); + +GFile * ostree_repo_get_pack_data_path (OstreeRepo *self, + const char *checksum); G_END_DECLS diff --git a/src/ostree/main.c b/src/ostree/main.c index 71a2efc8..1b434b66 100644 --- a/src/ostree/main.c +++ b/src/ostree/main.c @@ -42,10 +42,12 @@ static OstreeBuiltin builtins[] = { { "ls", ostree_builtin_ls, 0 }, { "prune", ostree_builtin_prune, 0 }, { "fsck", ostree_builtin_fsck, 0 }, + { "pack", ostree_builtin_pack, 0 }, { "remote", ostree_builtin_remote, 0 }, { "rev-parse", ostree_builtin_rev_parse, 0 }, { "remote", ostree_builtin_remote, 0 }, { "show", ostree_builtin_show, 0 }, + { "unpack", ostree_builtin_unpack, 0 }, { NULL } }; diff --git a/src/ostree/ostree-pull.c b/src/ostree/ostree-pull.c index 40f3d460..1e543569 100644 --- a/src/ostree/ostree-pull.c +++ b/src/ostree/ostree-pull.c @@ -54,6 +54,55 @@ log_verbose (const char *fmt, g_free (msg); } +typedef struct { + OstreeRepo *repo; + char *remote_name; + SoupSession *session; + SoupURI *base_uri; + + gboolean fetched_packs; + GPtrArray *cached_pack_indexes; +} OtPullData; + +static SoupURI * +suburi_new (SoupURI *base, + const char *first, + ...) G_GNUC_NULL_TERMINATED; + +static SoupURI * +suburi_new (SoupURI *base, + const char *first, + ...) +{ + va_list args; + GPtrArray *arg_array; + const char *arg; + char *subpath; + SoupURI *ret; + + arg_array = g_ptr_array_new (); + g_ptr_array_add (arg_array, (char*)soup_uri_get_path (base)); + g_ptr_array_add (arg_array, (char*)first); + + va_start (args, first); + + while ((arg = va_arg (args, const char *)) != NULL) + g_ptr_array_add (arg_array, (char*)arg); + g_ptr_array_add (arg_array, NULL); + + subpath = g_build_filenamev ((char**)arg_array->pdata); + g_ptr_array_unref (arg_array); + + ret = soup_uri_copy (base); + soup_uri_set_path (ret, subpath); + g_free (subpath); + + va_end (args); + + return ret; +} + + typedef struct { SoupSession *session; GOutputStream *stream; @@ -78,8 +127,7 @@ on_got_chunk (SoupMessage *msg, } static gboolean -fetch_uri (OstreeRepo *repo, - SoupSession *soup, +fetch_uri (OtPullData *pull_data, SoupURI *uri, const char *tmp_prefix, GFile **out_temp_filename, @@ -94,14 +142,14 @@ fetch_uri (OstreeRepo *repo, GOutputStream *output_stream = NULL; OstreeSoupChunkData chunkdata; - if (!ostree_create_temp_regular_file (ostree_repo_get_tmpdir (repo), + if (!ostree_create_temp_regular_file (ostree_repo_get_tmpdir (pull_data->repo), tmp_prefix, NULL, &ret_temp_filename, &output_stream, NULL, error)) goto out; - chunkdata.session = soup; + chunkdata.session = pull_data->session; chunkdata.stream = output_stream; chunkdata.had_error = FALSE; chunkdata.error = error; @@ -114,7 +162,7 @@ fetch_uri (OstreeRepo *repo, g_signal_connect (msg, "got-chunk", G_CALLBACK (on_got_chunk), &chunkdata); - response = soup_session_send_message (soup, msg); + response = soup_session_send_message (pull_data->session, msg); if (response != 200) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, @@ -138,8 +186,7 @@ fetch_uri (OstreeRepo *repo, } static gboolean -fetch_uri_contents_utf8 (OstreeRepo *repo, - SoupSession *soup, +fetch_uri_contents_utf8 (OtPullData *pull_data, SoupURI *uri, char **out_contents, GCancellable *cancellable, @@ -150,7 +197,7 @@ fetch_uri_contents_utf8 (OstreeRepo *repo, char *ret_contents = NULL; gsize len; - if (!fetch_uri (repo, soup, uri, "tmp-", &tmpf, cancellable, error)) + if (!fetch_uri (pull_data, uri, "tmp-", &tmpf, cancellable, error)) goto out; if (!g_file_load_contents (tmpf, cancellable, &ret_contents, &len, NULL, error)) @@ -173,29 +220,209 @@ fetch_uri_contents_utf8 (OstreeRepo *repo, return ret; } +static gboolean +fetch_one_pack_file (OtPullData *pull_data, + const char *pack_checksum, + GFile **out_cached_path, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GFile *ret_cached_path = NULL; + GFile *tmp_path = NULL; + char *pack_name = NULL; + SoupURI *pack_uri = NULL; + + if (!ostree_repo_get_cached_remote_pack_data (pull_data->repo, pull_data->remote_name, + pack_checksum, &ret_cached_path, + cancellable, error)) + goto out; + + if (ret_cached_path == NULL) + { + pack_name = g_strconcat ("ostpack-", pack_checksum, ".data", NULL); + pack_uri = suburi_new (pull_data->base_uri, "objects", "pack", pack_name, NULL); + + if (!fetch_uri (pull_data, pack_uri, "packdata-", &tmp_path, cancellable, error)) + goto out; + + if (!ostree_repo_take_cached_remote_pack_data (pull_data->repo, pull_data->remote_name, + pack_checksum, tmp_path, + cancellable, error)) + goto out; + } + + if (!ostree_repo_get_cached_remote_pack_data (pull_data->repo, pull_data->remote_name, + pack_checksum, &ret_cached_path, + cancellable, error)) + goto out; + + g_assert (ret_cached_path != NULL); + + ret = TRUE; + ot_transfer_out_value (out_cached_path, &ret_cached_path); + out: + g_clear_object (&ret_cached_path); + g_clear_object (&tmp_path); + g_free (pack_name); + if (pack_uri) + soup_uri_free (pack_uri); + return ret; +} static gboolean -fetch_object (OstreeRepo *repo, - SoupSession *soup, - SoupURI *baseuri, - const char *checksum, - OstreeObjectType objtype, - GFile **out_temp_path, - GCancellable *cancellable, - GError **error) +find_object_in_remote_packs (OtPullData *pull_data, + const char *checksum, + OstreeObjectType objtype, + char **out_pack_checksum, + guint64 *out_offset, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GVariant *mapped_pack = NULL; + GVariant *csum_bytes = NULL; + char *ret_pack_checksum = NULL; + guint64 offset; + guint i; + + csum_bytes = ostree_checksum_to_bytes (checksum); + + for (i = 0; i < pull_data->cached_pack_indexes->len; i++) + { + const char *pack_checksum = pull_data->cached_pack_indexes->pdata[i]; + + ot_clear_gvariant (&mapped_pack); + if (!ostree_repo_map_cached_remote_pack_index (pull_data->repo, pull_data->remote_name, + pack_checksum, &mapped_pack, + cancellable, error)) + goto out; + + if (ostree_pack_index_search (mapped_pack, csum_bytes, objtype, &offset)) + { + ret_pack_checksum = g_strdup (pack_checksum); + break; + } + } + + ret = TRUE; + ot_transfer_out_value (out_pack_checksum, &ret_pack_checksum); + if (out_offset) + *out_offset = offset; + out: + ot_clear_gvariant (&mapped_pack); + g_free (ret_pack_checksum); + ot_clear_gvariant (&csum_bytes); + return ret; +} + +static gboolean +fetch_one_cache_index (OtPullData *pull_data, + const char *pack_checksum, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + SoupURI *index_uri = NULL; + GFile *tmp_path = NULL; + char *pack_index_name = NULL; + + pack_index_name = g_strconcat ("ostpack-", pack_checksum, ".index", NULL); + index_uri = suburi_new (pull_data->base_uri, "objects", "pack", pack_index_name, NULL); + + if (!fetch_uri (pull_data, index_uri, "packindex-", &tmp_path, + cancellable, error)) + goto out; + + if (!ostree_repo_add_cached_remote_pack_index (pull_data->repo, pull_data->remote_name, + pack_checksum, tmp_path, + cancellable, error)) + goto out; + + if (!ot_gfile_unlink (tmp_path, cancellable, error)) + goto out; + + g_clear_object (&tmp_path); + + ret = TRUE; + out: + if (tmp_path != NULL) + (void) ot_gfile_unlink (tmp_path, NULL, NULL); + g_clear_object (&tmp_path); + g_free (pack_index_name); + if (index_uri) + soup_uri_free (index_uri); + return ret; +} + +static gboolean +fetch_and_cache_pack_indexes (OtPullData *pull_data, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + SoupURI *superindex_uri = NULL; + GFile *superindex_tmppath = NULL; + GPtrArray *cached_indexes = NULL; + GPtrArray *uncached_indexes = NULL; + GVariant *superindex_variant = NULL; + GVariantIter *contents_iter = NULL; + guint i; + + superindex_uri = suburi_new (pull_data->base_uri, "objects", "pack", "index", NULL); + + if (!fetch_uri (pull_data, superindex_uri, "index-", + &superindex_tmppath, cancellable, error)) + goto out; + + if (!ostree_repo_resync_cached_remote_pack_indexes (pull_data->repo, pull_data->remote_name, + superindex_tmppath, + &cached_indexes, &uncached_indexes, + cancellable, error)) + goto out; + + for (i = 0; i < cached_indexes->len; i++) + g_ptr_array_add (pull_data->cached_pack_indexes, + g_strdup (cached_indexes->pdata[i])); + + for (i = 0; i < uncached_indexes->len; i++) + { + const char *pack_checksum = uncached_indexes->pdata[i]; + + if (!fetch_one_cache_index (pull_data, pack_checksum, cancellable, error)) + goto out; + + g_ptr_array_add (pull_data->cached_pack_indexes, g_strdup (pack_checksum)); + } + + ret = TRUE; + out: + if (superindex_uri) + soup_uri_free (superindex_uri); + g_clear_object (&superindex_tmppath); + ot_clear_gvariant (&superindex_variant); + if (contents_iter) + g_variant_iter_free (contents_iter); + return ret; +} + +static gboolean +fetch_loose_object (OtPullData *pull_data, + const char *checksum, + OstreeObjectType objtype, + GFile **out_temp_path, + GCancellable *cancellable, + GError **error) { gboolean ret = FALSE; char *objpath = NULL; - char *relpath = NULL; SoupURI *obj_uri = NULL; GFile *ret_temp_path = NULL; objpath = ostree_get_relative_object_path (checksum, objtype); - obj_uri = soup_uri_copy (baseuri); - relpath = g_build_filename (soup_uri_get_path (obj_uri), objpath, NULL); - soup_uri_set_path (obj_uri, relpath); + obj_uri = suburi_new (pull_data->base_uri, objpath, NULL); - if (!fetch_uri (repo, soup, obj_uri, ostree_object_type_to_string (objtype), &ret_temp_path, + if (!fetch_uri (pull_data, obj_uri, ostree_object_type_to_string (objtype), &ret_temp_path, cancellable, error)) goto out; @@ -206,93 +433,321 @@ fetch_object (OstreeRepo *repo, soup_uri_free (obj_uri); g_clear_object (&ret_temp_path); g_free (objpath); - g_free (relpath); return ret; } static gboolean -fetch_and_store_object (OstreeRepo *repo, - SoupSession *soup, - SoupURI *baseuri, - const char *checksum, +find_object (OtPullData *pull_data, + const char *checksum, + OstreeObjectType objtype, + gboolean *out_is_stored, + gboolean *out_is_pending, + char **out_remote_pack_checksum, + guint64 *out_offset, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + gboolean ret_is_stored; + gboolean ret_is_pending; + GFile *stored_path = NULL; + GFile *pending_path = NULL; + char *local_pack_checksum = NULL; + char *ret_remote_pack_checksum = NULL; + guint64 offset; + + if (!ostree_repo_find_object (pull_data->repo, objtype, checksum, + &stored_path, &pending_path, + &local_pack_checksum, NULL, + cancellable, error)) + goto out; + + ret_is_stored = (stored_path != NULL || local_pack_checksum != NULL); + ret_is_pending = pending_path != NULL; + + if (!(ret_is_stored || ret_is_pending)) + { + if (!find_object_in_remote_packs (pull_data, checksum, objtype, + &ret_remote_pack_checksum, &offset, + cancellable, error)) + goto out; + } + + ret = TRUE; + if (out_is_stored) + *out_is_stored = ret_is_stored; + if (out_is_pending) + *out_is_pending = ret_is_pending; + ot_transfer_out_value (out_remote_pack_checksum, &ret_remote_pack_checksum); + if (out_offset) + *out_offset = offset; + out: + g_free (local_pack_checksum); + g_free (ret_remote_pack_checksum); + g_clear_object (&stored_path); + return ret; +} + +static void +unlink_file_on_unref (GFile *f) +{ + (void) ot_gfile_unlink (f, NULL, NULL); + g_object_unref (f); +} + +static gboolean +fetch_object_if_not_stored (OtPullData *pull_data, + const char *checksum, + OstreeObjectType objtype, + gboolean *out_is_stored, + gboolean *out_is_pending, + GInputStream **out_input, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + gboolean ret_is_stored = FALSE; + gboolean ret_is_pending = FALSE; + GInputStream *ret_input = NULL; + GFile *temp_path = NULL; + GFile *pack_path = NULL; + GMappedFile *pack_map = NULL; + char *remote_pack_checksum = NULL; + guint64 pack_offset = 0; + GVariant *pack_entry = NULL; + + if (!find_object (pull_data, checksum, objtype, &ret_is_stored, + &ret_is_pending, &remote_pack_checksum, + &pack_offset, cancellable, error)) + goto out; + + if (remote_pack_checksum != NULL) + { + g_assert (!(ret_is_stored || ret_is_pending)); + + if (!fetch_one_pack_file (pull_data, remote_pack_checksum, &pack_path, + cancellable, error)) + goto out; + + pack_map = g_mapped_file_new (ot_gfile_get_path_cached (pack_path), FALSE, error); + if (!pack_map) + goto out; + + if (!ostree_read_pack_entry_raw ((guchar*)g_mapped_file_get_contents (pack_map), + g_mapped_file_get_length (pack_map), + pack_offset, FALSE, &pack_entry, + cancellable, error)) + goto out; + + ret_input = ostree_read_pack_entry_as_stream (pack_entry); + g_object_set_data_full ((GObject*)ret_input, "ostree-pull-pack-map", + pack_map, (GDestroyNotify) g_mapped_file_unref); + pack_map = NULL; /* Transfer ownership */ + } + else if (!(ret_is_stored || ret_is_pending)) + { + if (!fetch_loose_object (pull_data, checksum, objtype, &temp_path, cancellable, error)) + goto out; + + ret_input = (GInputStream*)g_file_read (temp_path, cancellable, error); + if (!ret_input) + goto out; + g_object_set_data_full ((GObject*)ret_input, "ostree-tmpfile-unlink", + g_object_ref (temp_path), + (GDestroyNotify)unlink_file_on_unref); + } + + ret = TRUE; + ot_transfer_out_value (out_input, &ret_input); + if (out_is_stored) + *out_is_stored = ret_is_stored; + if (out_is_pending) + *out_is_pending = ret_is_pending; + out: + g_clear_object (&temp_path); + g_clear_object (&pack_path); + if (pack_map) + g_mapped_file_unref (pack_map); + ot_clear_gvariant (&pack_entry); + g_clear_object (&pack_path); + g_clear_object (&ret_input); + return ret; +} + +static gboolean +fetch_and_store_object (OtPullData *pull_data, + const char *checksum, OstreeObjectType objtype, - gboolean *out_is_pending, - GVariant **out_metadata, - GCancellable *cancellable, - GError **error) + gboolean *out_was_stored, + GCancellable *cancellable, + GError **error) { gboolean ret = FALSE; GFileInfo *file_info = NULL; GInputStream *input = NULL; GFile *stored_path = NULL; GFile *pending_path = NULL; - GFile *temp_path = NULL; - GVariant *ret_metadata = NULL; - gboolean ret_is_pending; + char *pack_checksum = NULL; + gboolean is_stored; + gboolean is_pending; g_assert (objtype != OSTREE_OBJECT_TYPE_RAW_FILE); - if (!ostree_repo_find_object (repo, objtype, checksum, - &stored_path, &pending_path, NULL, error)) + if (!fetch_object_if_not_stored (pull_data, checksum, objtype, + &is_stored, &is_pending, &input, + cancellable, error)) goto out; - - if (!(stored_path || pending_path)) - { - if (!fetch_object (repo, soup, baseuri, checksum, objtype, &temp_path, cancellable, error)) - goto out; - } - if (temp_path) + if (is_pending || input) { - file_info = g_file_query_info (temp_path, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, cancellable, error); - if (!file_info) - goto out; - - input = (GInputStream*)g_file_read (temp_path, cancellable, error); - if (!input) - goto out; - } - - if (pending_path || temp_path) - { - if (!ostree_repo_stage_object (repo, objtype, checksum, file_info, NULL, input, cancellable, error)) + if (!ostree_repo_stage_object (pull_data->repo, objtype, checksum, NULL, NULL, + input, cancellable, error)) goto out; log_verbose ("Staged object: %s.%s", checksum, ostree_object_type_to_string (objtype)); - - ret_is_pending = TRUE; - if (out_metadata) - { - if (!ostree_map_metadata_file (pending_path ? pending_path : temp_path, objtype, &ret_metadata, error)) - goto out; - } - } - else - { - ret_is_pending = FALSE; } ret = TRUE; - if (out_is_pending) - *out_is_pending = ret_is_pending; - ot_transfer_out_value (out_metadata, &ret_metadata); + if (out_was_stored) + *out_was_stored = is_stored; out: - if (temp_path) - (void) unlink (ot_gfile_get_path_cached (temp_path)); - ot_clear_gvariant (&ret_metadata); - g_clear_object (&temp_path); g_clear_object (&file_info); g_clear_object (&input); g_clear_object (&stored_path); g_clear_object (&pending_path); + g_free (pack_checksum); return ret; } static gboolean -fetch_and_store_tree_recurse (OstreeRepo *repo, - SoupSession *soup, - SoupURI *base_uri, +fetch_and_store_metadata (OtPullData *pull_data, + const char *checksum, + OstreeObjectType objtype, + gboolean *out_was_stored, + GVariant **out_variant, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + gboolean ret_was_stored; + GVariant *ret_variant = NULL; + + if (!fetch_and_store_object (pull_data, checksum, objtype, + &ret_was_stored, cancellable, error)) + goto out; + + if (!ostree_repo_load_variant (pull_data->repo, objtype, checksum, + &ret_variant, error)) + goto out; + + ret = TRUE; + ot_transfer_out_value (out_variant, &ret_variant); + if (out_was_stored) + *out_was_stored = ret_was_stored; + out: + ot_clear_gvariant (&ret_variant); + return ret; +} + +static gboolean +fetch_and_store_file (OtPullData *pull_data, + const char *checksum, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GInputStream *input = NULL; + GFile *stored_path = NULL; + GFile *pending_path = NULL; + char *pack_checksum = NULL; + GVariant *archive_metadata_container = NULL; + GVariant *archive_metadata = NULL; + GFileInfo *archive_file_info = NULL; + GVariant *archive_xattrs = NULL; + gboolean skip_archive_fetch; + + /* If we're fetching from an archive into a bare repository, we need + * to explicitly check for raw file types locally. + */ + if (ostree_repo_get_mode (pull_data->repo) == OSTREE_REPO_MODE_BARE) + { + if (!ostree_repo_find_object (pull_data->repo, OSTREE_OBJECT_TYPE_RAW_FILE, + checksum, &stored_path, &pending_path, &pack_checksum, + NULL, cancellable, error)) + goto out; + + if (stored_path || pack_checksum) + skip_archive_fetch = TRUE; + else if (pending_path != NULL) + { + skip_archive_fetch = TRUE; + if (!ostree_repo_stage_object (pull_data->repo, OSTREE_OBJECT_TYPE_RAW_FILE, + checksum, NULL, NULL, NULL, cancellable, error)) + goto out; + } + else + skip_archive_fetch = FALSE; + + g_clear_object (&stored_path); + } + else + { + skip_archive_fetch = FALSE; + } + + if (!skip_archive_fetch) + { + if (!fetch_object_if_not_stored (pull_data, checksum, + OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, + NULL, NULL, &input, cancellable, error)) + goto out; + + if (input != NULL) + { + if (!ot_util_variant_from_stream (input, OSTREE_SERIALIZED_VARIANT_FORMAT, + FALSE, &archive_metadata_container, cancellable, error)) + goto out; + + if (!ostree_unwrap_metadata (archive_metadata_container, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, + &archive_metadata, error)) + goto out; + + if (!ostree_parse_archived_file_meta (archive_metadata, &archive_file_info, + &archive_xattrs, error)) + goto out; + + g_clear_object (&input); + if (g_file_info_get_file_type (archive_file_info) == G_FILE_TYPE_REGULAR) + { + if (!fetch_object_if_not_stored (pull_data, checksum, + OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, + NULL, NULL, &input, + cancellable, error)) + goto out; + } + + if (!ostree_repo_stage_object (pull_data->repo, OSTREE_OBJECT_TYPE_RAW_FILE, checksum, + archive_file_info, archive_xattrs, input, + cancellable, error)) + goto out; + } + } + + ret = TRUE; + out: + g_free (pack_checksum); + g_clear_object (&stored_path); + g_clear_object (&pending_path); + g_clear_object (&input); + ot_clear_gvariant (&archive_metadata_container); + ot_clear_gvariant (&archive_metadata); + ot_clear_gvariant (&archive_xattrs); + g_clear_object (&archive_file_info); + return ret; +} + +static gboolean +fetch_and_store_tree_recurse (OtPullData *pull_data, const char *rev, GCancellable *cancellable, GError **error) @@ -301,22 +756,17 @@ fetch_and_store_tree_recurse (OstreeRepo *repo, GVariant *tree = NULL; GVariant *files_variant = NULL; GVariant *dirs_variant = NULL; - gboolean is_pending; + gboolean was_stored; int i, n; - GVariant *archive_metadata = NULL; - GFileInfo *archive_file_info = NULL; - GVariant *archive_xattrs = NULL; - GFile *meta_temp_path = NULL; - GFile *content_temp_path = NULL; GFile *stored_path = NULL; GFile *pending_path = NULL; - GInputStream *input = NULL; + char *pack_checksum = NULL; - if (!fetch_and_store_object (repo, soup, base_uri, rev, OSTREE_OBJECT_TYPE_DIR_TREE, - &is_pending, &tree, cancellable, error)) + if (!fetch_and_store_metadata (pull_data, rev, OSTREE_OBJECT_TYPE_DIR_TREE, + &was_stored, &tree, cancellable, error)) goto out; - if (!is_pending) + if (was_stored) log_verbose ("Already have tree %s", rev); else { @@ -337,81 +787,8 @@ fetch_and_store_tree_recurse (OstreeRepo *repo, if (!ostree_validate_checksum_string (checksum, error)) goto out; - g_clear_object (&stored_path); - g_clear_object (&pending_path); - /* If we're fetching from an archive into a bare repository, we need - * to explicitly check for raw file types locally. - */ - if (ostree_repo_get_mode (repo) == OSTREE_REPO_MODE_BARE) - { - if (!ostree_repo_find_object (repo, OSTREE_OBJECT_TYPE_RAW_FILE, checksum, - &stored_path, &pending_path, cancellable, error)) - goto out; - } - else - { - if (!ostree_repo_find_object (repo, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, checksum, - &stored_path, &pending_path, cancellable, error)) - goto out; - } - - g_clear_object (&input); - g_clear_object (&archive_file_info); - ot_clear_gvariant (&archive_xattrs); - if (!(stored_path || pending_path)) - { - g_clear_object (&meta_temp_path); - if (!fetch_object (repo, soup, base_uri, checksum, - OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, - &meta_temp_path, - cancellable, - error)) - goto out; - - if (!ostree_map_metadata_file (meta_temp_path, - OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, - &archive_metadata, error)) - goto out; - - if (!ostree_parse_archived_file_meta (archive_metadata, &archive_file_info, &archive_xattrs, error)) - goto out; - - if (g_file_info_get_file_type (archive_file_info) == G_FILE_TYPE_REGULAR) - { - if (!fetch_object (repo, soup, base_uri, checksum, - OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, - &content_temp_path, - cancellable, - error)) - goto out; - - input = (GInputStream*)g_file_read (content_temp_path, cancellable, error); - if (!input) - goto out; - } - } - - if (!stored_path) - { - log_verbose ("Staged file object: %s", checksum); - - if (!ostree_repo_stage_object (repo, OSTREE_OBJECT_TYPE_RAW_FILE, - checksum, - archive_file_info, archive_xattrs, input, - cancellable, error)) - goto out; - } - - if (meta_temp_path) - { - (void) unlink (ot_gfile_get_path_cached (meta_temp_path)); - g_clear_object (&meta_temp_path); - } - if (content_temp_path) - { - (void) unlink (ot_gfile_get_path_cached (content_temp_path)); - g_clear_object (&content_temp_path); - } + if (!fetch_and_store_file (pull_data, checksum, cancellable, error)) + goto out; } n = g_variant_n_children (dirs_variant); @@ -431,11 +808,11 @@ fetch_and_store_tree_recurse (OstreeRepo *repo, if (!ostree_validate_checksum_string (meta_checksum, error)) goto out; - if (!fetch_and_store_object (repo, soup, base_uri, meta_checksum, OSTREE_OBJECT_TYPE_DIR_META, - NULL, NULL, cancellable, error)) + if (!fetch_and_store_object (pull_data, meta_checksum, OSTREE_OBJECT_TYPE_DIR_META, + NULL, cancellable, error)) goto out; - if (!fetch_and_store_tree_recurse (repo, soup, base_uri, tree_checksum, cancellable, error)) + if (!fetch_and_store_tree_recurse (pull_data, tree_checksum, cancellable, error)) goto out; } } @@ -445,29 +822,14 @@ fetch_and_store_tree_recurse (OstreeRepo *repo, ot_clear_gvariant (&tree); ot_clear_gvariant (&files_variant); ot_clear_gvariant (&dirs_variant); - ot_clear_gvariant (&archive_metadata); - ot_clear_gvariant (&archive_xattrs); - g_clear_object (&archive_file_info); - g_clear_object (&input); g_clear_object (&stored_path); g_clear_object (&pending_path); - if (content_temp_path) - { - (void) unlink (ot_gfile_get_path_cached (content_temp_path)); - g_clear_object (&content_temp_path); - } - if (meta_temp_path) - { - (void) unlink (ot_gfile_get_path_cached (meta_temp_path)); - g_clear_object (&meta_temp_path); - } + g_free (pack_checksum); return ret; } static gboolean -fetch_and_store_commit_recurse (OstreeRepo *repo, - SoupSession *soup, - SoupURI *base_uri, +fetch_and_store_commit_recurse (OtPullData *pull_data, const char *rev, GCancellable *cancellable, GError **error) @@ -476,13 +838,13 @@ fetch_and_store_commit_recurse (OstreeRepo *repo, GVariant *commit = NULL; const char *tree_contents_checksum; const char *tree_meta_checksum; - gboolean is_pending; + gboolean was_stored; - if (!fetch_and_store_object (repo, soup, base_uri, rev, OSTREE_OBJECT_TYPE_COMMIT, - &is_pending, &commit, cancellable, error)) + if (!fetch_and_store_metadata (pull_data, rev, OSTREE_OBJECT_TYPE_COMMIT, + &was_stored, &commit, cancellable, error)) goto out; - if (!is_pending) + if (was_stored) log_verbose ("Already have commit %s", rev); else { @@ -490,11 +852,11 @@ fetch_and_store_commit_recurse (OstreeRepo *repo, g_variant_get_child (commit, 6, "&s", &tree_contents_checksum); g_variant_get_child (commit, 7, "&s", &tree_meta_checksum); - if (!fetch_and_store_object (repo, soup, base_uri, tree_meta_checksum, OSTREE_OBJECT_TYPE_DIR_META, - NULL, NULL, cancellable, error)) + if (!fetch_and_store_object (pull_data, tree_meta_checksum, OSTREE_OBJECT_TYPE_DIR_META, + NULL, cancellable, error)) goto out; - if (!fetch_and_store_tree_recurse (repo, soup, base_uri, tree_contents_checksum, + if (!fetch_and_store_tree_recurse (pull_data, tree_contents_checksum, cancellable, error)) goto out; } @@ -506,9 +868,7 @@ fetch_and_store_commit_recurse (OstreeRepo *repo, } static gboolean -fetch_ref_contents (OstreeRepo *repo, - SoupSession *soup, - SoupURI *base_uri, +fetch_ref_contents (OtPullData *pull_data, const char *ref, char **out_contents, GCancellable *cancellable, @@ -516,14 +876,11 @@ fetch_ref_contents (OstreeRepo *repo, { gboolean ret = FALSE; char *ret_contents = NULL; - char *refpath = NULL; SoupURI *target_uri = NULL; - target_uri = soup_uri_copy (base_uri); - refpath = g_build_filename (soup_uri_get_path (target_uri), "refs", "heads", ref, NULL); - soup_uri_set_path (target_uri, refpath); + target_uri = suburi_new (pull_data->base_uri, "refs", "heads", ref, NULL); - if (!fetch_uri_contents_utf8 (repo, soup, target_uri, &ret_contents, cancellable, error)) + if (!fetch_uri_contents_utf8 (pull_data, target_uri, &ret_contents, cancellable, error)) goto out; g_strchomp (ret_contents); @@ -534,7 +891,6 @@ fetch_ref_contents (OstreeRepo *repo, ret = TRUE; ot_transfer_out_value (out_contents, &ret_contents); out: - g_free (refpath); g_free (ret_contents); if (target_uri) soup_uri_free (target_uri); @@ -542,12 +898,9 @@ fetch_ref_contents (OstreeRepo *repo, } static gboolean -pull_one_commit (OstreeRepo *repo, - const char *remote, +pull_one_commit (OtPullData *pull_data, const char *branch, const char *rev, - SoupSession *soup, - SoupURI *base_uri, GCancellable *cancellable, GError **error) { @@ -557,9 +910,9 @@ pull_one_commit (OstreeRepo *repo, char *baseurl = NULL; char *original_rev = NULL; - remote_ref = g_strdup_printf ("%s/%s", remote, branch); + remote_ref = g_strdup_printf ("%s/%s", pull_data->remote_name, branch); - if (!ostree_repo_resolve_rev (repo, remote_ref, TRUE, &original_rev, error)) + if (!ostree_repo_resolve_rev (pull_data->repo, remote_ref, TRUE, &original_rev, error)) goto out; if (original_rev && strcmp (rev, original_rev) == 0) @@ -571,16 +924,27 @@ pull_one_commit (OstreeRepo *repo, if (!ostree_validate_checksum_string (rev, error)) goto out; - if (!ostree_repo_prepare_transaction (repo, NULL, error)) + if (!pull_data->fetched_packs) + { + pull_data->fetched_packs = TRUE; + pull_data->cached_pack_indexes = g_ptr_array_new_with_free_func (g_free); + + g_print ("Fetching packs\n"); + + if (!fetch_and_cache_pack_indexes (pull_data, cancellable, error)) + goto out; + } + + if (!ostree_repo_prepare_transaction (pull_data->repo, NULL, error)) goto out; - if (!fetch_and_store_commit_recurse (repo, soup, base_uri, rev, cancellable, error)) + if (!fetch_and_store_commit_recurse (pull_data, rev, cancellable, error)) goto out; - if (!ostree_repo_commit_transaction (repo, cancellable, error)) + if (!ostree_repo_commit_transaction (pull_data->repo, cancellable, error)) goto out; - if (!ostree_repo_write_ref (repo, remote, branch, rev, error)) + if (!ostree_repo_write_ref (pull_data->repo, pull_data->remote_name, branch, rev, error)) goto out; g_print ("remote %s is now %s\n", remote_ref, rev); @@ -656,9 +1020,9 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error) { GOptionContext *context; gboolean ret = FALSE; + OtPullData pull_data_real; + OtPullData *pull_data = &pull_data_real; OstreeRepo *repo = NULL; - const char *remote; - SoupSession *soup = NULL; char *path = NULL; char *baseurl = NULL; char *summary_data = NULL; @@ -682,27 +1046,29 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error) if (!ostree_repo_check (repo, error)) goto out; + memset (pull_data, 0, sizeof (*pull_data)); + pull_data->repo = repo; + if (argc < 2) { ot_util_usage_error (context, "REMOTE must be specified", error); goto out; } - remote = argv[1]; - - soup = soup_session_sync_new_with_options (SOUP_SESSION_USER_AGENT, "ostree ", - SOUP_SESSION_ADD_FEATURE_BY_TYPE, SOUP_TYPE_COOKIE_JAR, - NULL); + pull_data->remote_name = g_strdup (argv[1]); + pull_data->session = soup_session_sync_new_with_options (SOUP_SESSION_USER_AGENT, "ostree ", + SOUP_SESSION_ADD_FEATURE_BY_TYPE, SOUP_TYPE_COOKIE_JAR, + NULL); config = ostree_repo_get_config (repo); - key = g_strdup_printf ("remote \"%s\"", remote); + key = g_strdup_printf ("remote \"%s\"", pull_data->remote_name); baseurl = g_key_file_get_string (config, key, "url", error); if (!baseurl) goto out; - base_uri = soup_uri_new (baseurl); + pull_data->base_uri = soup_uri_new (baseurl); - if (!base_uri) + if (!pull_data->base_uri) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Failed to parse url '%s'", baseurl); @@ -717,7 +1083,7 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error) const char *branch = argv[i]; char *contents; - if (!fetch_ref_contents (repo, soup, base_uri, branch, &contents, cancellable, error)) + if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error)) goto out; /* Transfer ownership of contents */ @@ -730,7 +1096,7 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error) path = g_build_filename (soup_uri_get_path (summary_uri), "refs", "summary", NULL); soup_uri_set_path (summary_uri, path); - if (!fetch_uri_contents_utf8 (repo, soup, summary_uri, &summary_data, cancellable, error)) + if (!fetch_uri_contents_utf8 (pull_data, summary_uri, &summary_data, cancellable, error)) goto out; if (!parse_ref_summary (summary_data, &refs_to_fetch, error)) @@ -744,7 +1110,7 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error) const char *ref = key; const char *sha256 = value; - if (!pull_one_commit (repo, remote, ref, sha256, soup, base_uri, cancellable, error)) + if (!pull_one_commit (pull_data, ref, sha256, cancellable, error)) goto out; } @@ -758,13 +1124,14 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error) g_free (branch_rev); if (context) g_option_context_free (context); - g_clear_object (&soup); - if (base_uri) - soup_uri_free (base_uri); + g_clear_object (&pull_data->session); + if (pull_data->base_uri) + soup_uri_free (pull_data->base_uri); + if (pull_data->cached_pack_indexes) + g_ptr_array_unref (pull_data->cached_pack_indexes); if (summary_uri) soup_uri_free (summary_uri); g_clear_object (&repo); - g_clear_object (&soup); return ret; } diff --git a/src/ostree/ot-builtin-fsck.c b/src/ostree/ot-builtin-fsck.c index 9671ffab..f4f567ae 100644 --- a/src/ostree/ot-builtin-fsck.c +++ b/src/ostree/ot-builtin-fsck.c @@ -26,6 +26,7 @@ #include "ostree.h" #include +#include static gboolean quiet; static gboolean delete; @@ -38,8 +39,8 @@ static GOptionEntry options[] = { typedef struct { OstreeRepo *repo; - guint n_objects; - gboolean had_error; + guint n_loose_objects; + guint n_pack_files; } OtFsckData; static gboolean @@ -123,62 +124,157 @@ checksum_archived_file (OtFsckData *data, return ret; } -static void -object_iter_callback (OstreeRepo *repo, - const char *exp_checksum, - OstreeObjectType objtype, - GFile *objf, - GFileInfo *file_info, - gpointer user_data) +static gboolean +fsck_loose_object (OtFsckData *data, + const char *exp_checksum, + OstreeObjectType objtype, + GCancellable *cancellable, + GError **error) { - OtFsckData *data = user_data; + gboolean ret = FALSE; + GFile *objf = NULL; GChecksum *real_checksum = NULL; - GError *error = NULL; - /* nlinks = g_file_info_get_attribute_uint32 (file_info, "unix::nlink"); - if (nlinks < 2 && !quiet) - g_printerr ("note: floating object: %s\n", path); */ + objf = ostree_repo_get_object_path (data->repo, exp_checksum, objtype); if (objtype == OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META) { if (!g_str_has_suffix (ot_gfile_get_path_cached (objf), ".archive-meta")) { - g_set_error (&error, G_IO_ERROR, G_IO_ERROR_FAILED, + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Invalid archive filename '%s'", ot_gfile_get_path_cached (objf)); goto out; } - if (!checksum_archived_file (data, exp_checksum, objf, &real_checksum, &error)) + if (!checksum_archived_file (data, exp_checksum, objf, &real_checksum, error)) goto out; } else if (objtype == OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT) ; /* Handled above */ else { - if (!ostree_checksum_file (objf, objtype, &real_checksum, NULL, &error)) + if (!ostree_checksum_file (objf, objtype, &real_checksum, NULL, error)) goto out; } if (real_checksum && strcmp (exp_checksum, g_checksum_get_string (real_checksum)) != 0) { - data->had_error = TRUE; - g_printerr ("ERROR: corrupted object '%s'; actual checksum: %s\n", - ot_gfile_get_path_cached (objf), g_checksum_get_string (real_checksum)); + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "corrupted loose object '%s'; actual checksum: %s", + ot_gfile_get_path_cached (objf), g_checksum_get_string (real_checksum)); if (delete) (void) unlink (ot_gfile_get_path_cached (objf)); + goto out; } - data->n_objects++; + data->n_loose_objects++; + ret = TRUE; out: ot_clear_checksum (&real_checksum); - if (error != NULL) - { - g_printerr ("%s\n", error->message); - g_clear_error (&error); - } + return ret; } +static gboolean +fsck_pack_files (OtFsckData *data, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GPtrArray *pack_indexes = NULL; + GVariant *index_variant = NULL; + GFile *pack_index_path = NULL; + GFile *pack_data_path = NULL; + GFileInfo *pack_info = NULL; + GInputStream *input = NULL; + GChecksum *pack_content_checksum = NULL; + GVariantIter *index_content_iter = NULL; + guint i; + guint32 objtype; + guint64 offset; + guint64 pack_size; + + if (!ostree_repo_list_pack_indexes (data->repo, &pack_indexes, cancellable, error)) + goto out; + + for (i = 0; i < pack_indexes->len; i++) + { + const char *checksum = pack_indexes->pdata[i]; + + g_clear_object (&pack_index_path); + pack_index_path = ostree_repo_get_pack_index_path (data->repo, checksum); + + ot_clear_gvariant (&index_variant); + if (!ot_util_variant_map (pack_index_path, + OSTREE_PACK_INDEX_VARIANT_FORMAT, + &index_variant, error)) + goto out; + + if (!ostree_validate_structureof_pack_index (index_variant, error)) + goto out; + + g_clear_object (&pack_data_path); + pack_data_path = ostree_repo_get_pack_data_path (data->repo, checksum); + + g_clear_object (&input); + input = (GInputStream*)g_file_read (pack_data_path, cancellable, error); + if (!input) + goto out; + + g_clear_object (&pack_info); + pack_info = g_file_input_stream_query_info ((GFileInputStream*)input, OSTREE_GIO_FAST_QUERYINFO, + cancellable, error); + if (!pack_info) + goto out; + pack_size = g_file_info_get_attribute_uint64 (pack_info, "standard::size"); + + if (pack_content_checksum) + g_checksum_free (pack_content_checksum); + if (!ot_gio_checksum_stream (input, &pack_content_checksum, cancellable, error)) + goto out; + + if (strcmp (g_checksum_get_string (pack_content_checksum), checksum) != 0) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "corrupted pack '%s', expected checksum %s", + checksum, g_checksum_get_string (pack_content_checksum)); + goto out; + } + + g_variant_get_child (index_variant, 2, "a(uayt)", &index_content_iter); + + while (g_variant_iter_loop (index_content_iter, "(u@ayt)", + &objtype, NULL, &offset)) + { + offset = GUINT64_FROM_BE (offset); + if (offset > pack_size) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "corrupted pack '%s', offset %" G_GUINT64_FORMAT " larger than file size %" G_GUINT64_FORMAT, + checksum, + offset, pack_size); + goto out; + } + } + + data->n_pack_files++; + } + + ret = TRUE; + out: + if (index_content_iter) + g_variant_iter_free (index_content_iter); + if (pack_content_checksum) + g_checksum_free (pack_content_checksum); + if (pack_indexes) + g_ptr_array_unref (pack_indexes); + g_clear_object (&pack_info); + g_clear_object (&pack_data_path); + g_clear_object (&input); + return ret; +} + + gboolean ostree_builtin_fsck (int argc, char **argv, GFile *repo_path, GError **error) { @@ -186,6 +282,10 @@ ostree_builtin_fsck (int argc, char **argv, GFile *repo_path, GError **error) OtFsckData data; gboolean ret = FALSE; OstreeRepo *repo = NULL; + GHashTable *objects = NULL; + GCancellable *cancellable = NULL; + GHashTableIter hash_iter; + gpointer key, value; context = g_option_context_new ("- Check the repository for consistency"); g_option_context_add_main_entries (context, options, NULL); @@ -197,26 +297,47 @@ ostree_builtin_fsck (int argc, char **argv, GFile *repo_path, GError **error) if (!ostree_repo_check (repo, error)) goto out; + memset (&data, 0, sizeof (data)); data.repo = repo; - data.n_objects = 0; - data.had_error = FALSE; - if (!ostree_repo_iter_objects (repo, object_iter_callback, &data, error)) + if (!ostree_repo_list_objects (repo, OSTREE_REPO_LIST_OBJECTS_ALL, + &objects, cancellable, error)) + goto out; + + g_hash_table_iter_init (&hash_iter, objects); + + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + GVariant *serialized_key = key; + GVariant *objdata = value; + const char *checksum; + OstreeObjectType objtype; + gboolean is_loose; + + ostree_object_name_deserialize (serialized_key, &checksum, &objtype); + + g_variant_get_child (objdata, 0, "b", &is_loose); + + if (is_loose) + { + if (!fsck_loose_object (&data, checksum, objtype, cancellable, error)) + goto out; + } + } + + if (!fsck_pack_files (&data, cancellable, error)) goto out; - if (data.had_error) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Encountered filesystem consistency errors"); - goto out; - } if (!quiet) - g_printerr ("Total Objects: %u\n", data.n_objects); + g_print ("Loose Objects: %u\n", data.n_loose_objects); + g_print ("Pack files: %u\n", data.n_pack_files); ret = TRUE; out: if (context) g_option_context_free (context); g_clear_object (&repo); + if (objects) + g_hash_table_unref (objects); return ret; } diff --git a/src/ostree/ot-builtin-init.c b/src/ostree/ot-builtin-init.c index 935bae70..5d648d89 100644 --- a/src/ostree/ot-builtin-init.c +++ b/src/ostree/ot-builtin-init.c @@ -45,7 +45,9 @@ ostree_builtin_init (int argc, char **argv, GFile *repo_path, GError **error) gboolean ret = FALSE; GFile *child = NULL; GFile *grandchild = NULL; + GCancellable *cancellable = NULL; GString *config_data = NULL; + OstreeRepo *repo = NULL; context = g_option_context_new ("- Initialize a new empty repository"); g_option_context_add_main_entries (context, options, NULL); @@ -63,38 +65,53 @@ ostree_builtin_init (int argc, char **argv, GFile *repo_path, GError **error) NULL, FALSE, 0, NULL, NULL, error)) goto out; - g_clear_object (&child); + g_clear_object (&child); child = g_file_get_child (repo_path, "objects"); if (!g_file_make_directory (child, NULL, error)) goto out; - g_clear_object (&child); + g_clear_object (&grandchild); + grandchild = g_file_get_child (child, "pack"); + if (!g_file_make_directory (grandchild, NULL, error)) + goto out; + + g_clear_object (&child); child = g_file_get_child (repo_path, "tmp"); if (!g_file_make_directory (child, NULL, error)) goto out; - g_clear_object (&child); + g_clear_object (&child); child = g_file_get_child (repo_path, "refs"); if (!g_file_make_directory (child, NULL, error)) goto out; + g_clear_object (&grandchild); grandchild = g_file_get_child (child, "heads"); if (!g_file_make_directory (grandchild, NULL, error)) goto out; - g_clear_object (&grandchild); + g_clear_object (&grandchild); grandchild = g_file_get_child (child, "remotes"); if (!g_file_make_directory (grandchild, NULL, error)) goto out; - g_clear_object (&grandchild); g_clear_object (&child); - child = g_file_get_child (repo_path, "tags"); if (!g_file_make_directory (child, NULL, error)) goto out; + g_clear_object (&child); + child = g_file_get_child (repo_path, "remote-cache"); + if (!g_file_make_directory (child, NULL, error)) + goto out; + + repo = ostree_repo_new (repo_path); + if (!ostree_repo_check (repo, error)) + goto out; + + if (!ostree_repo_regenerate_pack_index (repo, cancellable, error)) + goto out; ret = TRUE; out: @@ -104,5 +121,6 @@ ostree_builtin_init (int argc, char **argv, GFile *repo_path, GError **error) g_string_free (config_data, TRUE); g_clear_object (&child); g_clear_object (&grandchild); + g_clear_object (&repo); return ret; } diff --git a/src/ostree/ot-builtin-local-clone.c b/src/ostree/ot-builtin-local-clone.c index 87b01516..1c9a3d0e 100644 --- a/src/ostree/ot-builtin-local-clone.c +++ b/src/ostree/ot-builtin-local-clone.c @@ -97,23 +97,29 @@ copy_dir_contents_recurse (GFile *src, return ret; } -static void -object_iter_callback (OstreeRepo *repo, - const char *checksum, - OstreeObjectType objtype, - GFile *objfile, - GFileInfo *file_info, - gpointer user_data) +static gboolean +import_loose_object (OtLocalCloneData *data, + const char *checksum, + OstreeObjectType objtype, + GCancellable *cancellable, + GError **error) { - OtLocalCloneData *data = user_data; - GError *real_error = NULL; - GError **error = &real_error; + gboolean ret = FALSE; + GFile *objfile = NULL; + GFileInfo *file_info = NULL; GFile *content_path = NULL; GFileInfo *archive_info = NULL; GVariant *archive_metadata = NULL; GVariant *xattrs = NULL; GInputStream *input = NULL; + objfile = ostree_repo_get_object_path (data->src_repo, checksum, objtype); + file_info = g_file_query_info (objfile, OSTREE_GIO_FAST_QUERYINFO, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, cancellable, error); + + if (file_info == NULL) + goto out; + if (objtype == OSTREE_OBJECT_TYPE_RAW_FILE) xattrs = ostree_get_xattrs_for_file (objfile, error); @@ -121,13 +127,13 @@ object_iter_callback (OstreeRepo *repo, ; else if (objtype == OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META) { - if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, checksum, &archive_metadata, error)) + if (!ostree_repo_load_variant (data->src_repo, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, checksum, &archive_metadata, error)) goto out; if (!ostree_parse_archived_file_meta (archive_metadata, &archive_info, &xattrs, error)) goto out; - content_path = ostree_repo_get_object_path (repo, checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT); + content_path = ostree_repo_get_object_path (data->src_repo, checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT); if (g_file_info_get_file_type (archive_info) == G_FILE_TYPE_REGULAR) { @@ -136,8 +142,8 @@ object_iter_callback (OstreeRepo *repo, goto out; } - if (!ostree_repo_stage_object_trusted (data->dest_repo, OSTREE_OBJECT_TYPE_RAW_FILE, checksum, - archive_info, xattrs, input, + if (!ostree_repo_stage_object_trusted (data->dest_repo, OSTREE_OBJECT_TYPE_RAW_FILE, + checksum, FALSE, archive_info, xattrs, input, NULL, error)) goto out; } @@ -151,23 +157,21 @@ object_iter_callback (OstreeRepo *repo, } if (!ostree_repo_stage_object_trusted (data->dest_repo, objtype, checksum, - file_info, xattrs, input, + FALSE, file_info, xattrs, input, NULL, error)) goto out; } + ret = TRUE; out: ot_clear_gvariant (&archive_metadata); ot_clear_gvariant (&xattrs); g_clear_object (&archive_info); g_clear_object (&input); g_clear_object (&content_path); - if (real_error != NULL) - { - g_printerr ("%s\n", real_error->message); - g_clear_error (error); - exit (1); - } + g_clear_object (&file_info); + g_clear_object (&objfile); + return ret; } static gboolean @@ -209,6 +213,7 @@ ostree_builtin_local_clone (int argc, char **argv, GFile *repo_path, GError **er { gboolean ret = FALSE; GCancellable *cancellable = NULL; + GHashTable *objects = NULL; GOptionContext *context; const char *destination; GFile *dest_f = NULL; @@ -220,6 +225,8 @@ ostree_builtin_local_clone (int argc, char **argv, GFile *repo_path, GError **er GFile *src_dir = NULL; GFile *dest_dir = NULL; int i; + GHashTableIter hash_iter; + gpointer key, value; context = g_option_context_new ("DEST ... - Create new repository DEST"); g_option_context_add_main_entries (context, options, NULL); @@ -266,11 +273,33 @@ ostree_builtin_local_clone (int argc, char **argv, GFile *repo_path, GError **er data.uids_differ = g_file_info_get_attribute_uint32 (src_info, "unix::uid") != g_file_info_get_attribute_uint32 (dest_info, "unix::uid"); - if (!ostree_repo_prepare_transaction (data.dest_repo, NULL, error)) + if (!ostree_repo_list_objects (data.src_repo, OSTREE_REPO_LIST_OBJECTS_ALL, + &objects, cancellable, error)) goto out; - if (!ostree_repo_iter_objects (data.src_repo, object_iter_callback, &data, error)) + if (!ostree_repo_prepare_transaction (data.dest_repo, NULL, error)) goto out; + + g_hash_table_iter_init (&hash_iter, objects); + + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + GVariant *serialized_key = key; + GVariant *objdata = value; + const char *checksum; + OstreeObjectType objtype; + gboolean is_loose; + + ostree_object_name_deserialize (serialized_key, &checksum, &objtype); + + g_variant_get_child (objdata, 0, "b", &is_loose); + + if (is_loose) + { + if (!import_loose_object (&data, checksum, objtype, cancellable, error)) + goto out; + } + } if (!ostree_repo_commit_transaction (data.dest_repo, NULL, error)) goto out; @@ -311,5 +340,7 @@ ostree_builtin_local_clone (int argc, char **argv, GFile *repo_path, GError **er g_clear_object (&dest_dir); g_clear_object (&data.src_repo); g_clear_object (&data.dest_repo); + if (objects) + g_hash_table_unref (objects); return ret; } diff --git a/src/ostree/ot-builtin-pack.c b/src/ostree/ot-builtin-pack.c new file mode 100644 index 00000000..cdef0f0f --- /dev/null +++ b/src/ostree/ot-builtin-pack.c @@ -0,0 +1,920 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2012 Colin Walters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Author: Colin Walters + */ + +#include "config.h" + +#include "ot-builtins.h" +#include "ostree.h" + +#include +#include + +#include +#include + +#define OT_DEFAULT_PACK_SIZE_BYTES (50*1024*1024) +#define OT_GZIP_COMPRESSION_LEVEL (8) + +static gboolean opt_analyze_only; +static gboolean opt_reindex_only; +static gboolean opt_keep_loose; +static char* opt_pack_size; +static char* opt_int_compression; +static char* opt_ext_compression; + +typedef enum { + OT_COMPRESSION_NONE, + OT_COMPRESSION_GZIP, + OT_COMPRESSION_XZ +} OtCompressionType; + +static GOptionEntry options[] = { + { "pack-size", 0, 0, G_OPTION_ARG_STRING, &opt_pack_size, "Maximum uncompressed size of packfiles in bytes; may be suffixed with k, m, or g", "BYTES" }, + { "internal-compression", 0, 0, G_OPTION_ARG_STRING, &opt_int_compression, "Compress objects using COMPRESSION", "COMPRESSION" }, + { "external-compression", 0, 0, G_OPTION_ARG_STRING, &opt_ext_compression, "Compress entire packfiles using COMPRESSION", "COMPRESSION" }, + { "analyze-only", 0, 0, G_OPTION_ARG_NONE, &opt_analyze_only, "Just analyze current state", NULL }, + { "reindex-only", 0, 0, G_OPTION_ARG_NONE, &opt_reindex_only, "Regenerate pack index", NULL }, + { "keep-loose", 0, 0, G_OPTION_ARG_NONE, &opt_keep_loose, "Don't delete loose objects", NULL }, + { NULL } +}; + +typedef struct { + OstreeRepo *repo; + + guint64 pack_size; + OtCompressionType int_compression; + OtCompressionType ext_compression; + + gboolean had_error; + GError **error; +} OtRepackData; + +typedef struct { + GOutputStream *out; + GPtrArray *compressor_argv; + GPid compress_child_pid; +} OtBuildRepackFile; + +static gint +compare_object_data_by_size (gconstpointer ap, + gconstpointer bp) +{ + GVariant *a = *(void **)ap; + GVariant *b = *(void **)bp; + guint64 a_size; + guint64 b_size; + + g_variant_get_child (a, 2, "t", &a_size); + g_variant_get_child (b, 2, "t", &b_size); + if (a == b) + return 0; + else if (a > b) + return 1; + else + return -1; +} + +static gboolean +write_bytes_update_checksum (GOutputStream *output, + gconstpointer bytes, + gsize len, + GChecksum *checksum, + guint64 *inout_offset, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + gsize bytes_written; + + if (len > 0) + { + g_checksum_update (checksum, (guchar*) bytes, len); + if (!g_output_stream_write_all (output, bytes, len, &bytes_written, + cancellable, error)) + goto out; + g_assert_cmpint (bytes_written, ==, len); + *inout_offset += bytes_written; + } + + ret = TRUE; + out: + return ret; +} + +static gboolean +write_padding (GOutputStream *output, + guint alignment, + GChecksum *checksum, + guint64 *inout_offset, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + guint bits; + guint padding_len; + guchar padding_nuls[8] = {0, 0, 0, 0, 0, 0, 0, 0}; + + if (alignment == 8) + bits = ((*inout_offset) & 7); + else + bits = ((*inout_offset) & 3); + + if (bits > 0) + { + padding_len = alignment - bits; + if (!write_bytes_update_checksum (output, (guchar*)padding_nuls, padding_len, + checksum, inout_offset, cancellable, error)) + goto out; + } + + ret = TRUE; + out: + return ret; +} + +static gboolean +write_variant_with_size (GOutputStream *output, + GVariant *variant, + GChecksum *checksum, + guint64 *inout_offset, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + guint64 variant_size; + guint32 variant_size_u32_be; + + g_assert ((*inout_offset & 3) == 0); + + /* Write variant size */ + variant_size = g_variant_get_size (variant); + g_assert (variant_size < G_MAXUINT32); + variant_size_u32_be = GUINT32_TO_BE((guint32) variant_size); + + if (!write_bytes_update_checksum (output, (guchar*)&variant_size_u32_be, 4, + checksum, inout_offset, cancellable, error)) + goto out; + + /* Pad to offset of 8, write variant */ + if (!write_padding (output, 8, checksum, inout_offset, cancellable, error)) + goto out; + g_assert ((*inout_offset & 7) == 0); + + if (!write_bytes_update_checksum (output, g_variant_get_data (variant), + variant_size, checksum, + inout_offset, cancellable, error)) + goto out; + + ret = TRUE; + out: + return ret; +} + +static gint +compare_index_content (gconstpointer ap, + gconstpointer bp) +{ + gpointer a = *((gpointer*)ap); + gpointer b = *((gpointer*)bp); + GVariant *a_v = a; + GVariant *b_v = b; + GVariant *a_csum_bytes; + GVariant *b_csum_bytes; + guint32 a_objtype; + guint32 b_objtype; + guint64 a_offset; + guint64 b_offset; + int c; + + g_variant_get (a_v, "(u@ayt)", &a_objtype, &a_csum_bytes, &a_offset); + g_variant_get (b_v, "(u@ayt)", &b_objtype, &b_csum_bytes, &b_offset); + a_objtype = GUINT32_FROM_BE (a_objtype); + b_objtype = GUINT32_FROM_BE (b_objtype); + c = ostree_cmp_checksum_bytes (a_csum_bytes, b_csum_bytes); + if (c == 0) + { + if (a_objtype < b_objtype) + c = -1; + else if (a_objtype > b_objtype) + c = 1; + } + return c; +} + +static gboolean +delete_loose_object (OtRepackData *data, + const char *checksum, + OstreeObjectType objtype, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GFile *object_path = NULL; + GFile *content_object_path = NULL; + GVariant *archive_meta = NULL; + GFileInfo *file_info = NULL; + GVariant *xattrs = NULL; + + object_path = ostree_repo_get_object_path (data->repo, checksum, objtype); + + /* This is gross - we need to specially clean up symbolic link object content */ + if (objtype == OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META) + { + if (!ostree_map_metadata_file (object_path, objtype, &archive_meta, error)) + goto out; + if (!ostree_parse_archived_file_meta (archive_meta, &file_info, &xattrs, error)) + goto out; + + if (g_file_info_get_file_type (file_info) != G_FILE_TYPE_REGULAR) + { + content_object_path = ostree_repo_get_object_path (data->repo, checksum, + OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT); + if (!ot_gfile_unlink (content_object_path, cancellable, error)) + { + g_prefix_error (error, "Failed to delete archived content '%s'", + ot_gfile_get_path_cached (content_object_path)); + goto out; + } + } + } + + if (!ot_gfile_unlink (object_path, cancellable, error)) + { + g_prefix_error (error, "Failed to delete archived file metadata '%s'", + ot_gfile_get_path_cached (object_path)); + goto out; + } + + ret = TRUE; + out: + g_clear_object (&object_path); + g_clear_object (&content_object_path); + ot_clear_gvariant (&archive_meta); + g_clear_object (&file_info); + ot_clear_gvariant (&xattrs); + return ret; +} + +static gboolean +create_pack_file (OtRepackData *data, + GPtrArray *objects, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GFile *pack_dir = NULL; + GFile *index_temppath = NULL; + GOutputStream *index_out = NULL; + GFile *pack_temppath = NULL; + GOutputStream *pack_out = NULL; + GFile *object_path = NULL; + GFileInfo *object_file_info = NULL; + GFileInputStream *object_input = NULL; + GConverter *compressor = NULL; + GConverterInputStream *compressed_object_input = NULL; + guint i; + guint64 offset; + gsize bytes_written; + GPtrArray *index_content_list = NULL; + GVariant *pack_header = NULL; + GVariant *packed_object = NULL; + GVariant *index_content = NULL; + GVariantBuilder index_content_builder; + GChecksum *pack_checksum = NULL; + char *pack_name = NULL; + GFile *pack_file_path = NULL; + GFile *pack_index_path = NULL; + GMemoryOutputStream *object_data_stream = NULL; + + if (g_cancellable_set_error_if_cancelled (cancellable, error)) + return FALSE; + + if (!ostree_create_temp_regular_file (ostree_repo_get_tmpdir (data->repo), + "pack-index", NULL, + &index_temppath, + &index_out, + cancellable, error)) + goto out; + + if (!ostree_create_temp_regular_file (ostree_repo_get_tmpdir (data->repo), + "pack-content", NULL, + &pack_temppath, + &pack_out, + cancellable, error)) + goto out; + + index_content_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); + + offset = 0; + pack_checksum = g_checksum_new (G_CHECKSUM_SHA256); + + pack_header = g_variant_new ("(s@a{sv}t)", + "OSTv0PACKFILE", + g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0), + (guint64)objects->len); + + if (!write_variant_with_size (pack_out, pack_header, pack_checksum, &offset, + cancellable, error)) + goto out; + + for (i = 0; i < objects->len; i++) + { + GVariant *object_data = objects->pdata[i]; + const char *checksum; + guint32 objtype_u32; + OstreeObjectType objtype; + guint64 expected_objsize; + guint64 objsize; + GInputStream *read_object_in; + guchar entry_flags = 0; + GVariant *index_entry; + + g_variant_get (object_data, "(&sut)", &checksum, &objtype_u32, &expected_objsize); + + objtype = (OstreeObjectType) objtype_u32; + + switch (data->int_compression) + { + case OT_COMPRESSION_GZIP: + { + entry_flags |= OSTREE_PACK_FILE_ENTRY_FLAG_GZIP; + break; + } + default: + { + g_assert_not_reached (); + } + } + + g_clear_object (&object_path); + object_path = ostree_repo_get_object_path (data->repo, checksum, objtype); + + g_clear_object (&object_input); + object_input = g_file_read (object_path, cancellable, error); + if (!object_input) + goto out; + + g_clear_object (&object_file_info); + object_file_info = g_file_input_stream_query_info (object_input, OSTREE_GIO_FAST_QUERYINFO, cancellable, error); + if (!object_file_info) + goto out; + + objsize = g_file_info_get_attribute_uint64 (object_file_info, G_FILE_ATTRIBUTE_STANDARD_SIZE); + + g_assert_cmpint (objsize, ==, expected_objsize); + + g_clear_object (&object_data_stream); + object_data_stream = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free); + + if (entry_flags & OSTREE_PACK_FILE_ENTRY_FLAG_GZIP) + { + g_clear_object (&compressor); + compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP, OT_GZIP_COMPRESSION_LEVEL); + + g_clear_object (&compressed_object_input); + compressed_object_input = (GConverterInputStream*)g_object_new (G_TYPE_CONVERTER_INPUT_STREAM, + "converter", compressor, + "base-stream", object_input, + "close-base-stream", TRUE, + NULL); + read_object_in = (GInputStream*)compressed_object_input; + } + else + { + read_object_in = (GInputStream*)object_input; + } + + if (!g_output_stream_splice ((GOutputStream*)object_data_stream, read_object_in, + G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET, + cancellable, error)) + goto out; + + ot_clear_gvariant (&packed_object); + { + guchar *data = g_memory_output_stream_get_data (object_data_stream); + gsize data_len = g_memory_output_stream_get_data_size (object_data_stream); + packed_object = g_variant_new ("(uy@ay@ay)", GUINT32_TO_BE ((guint32)objtype), + entry_flags, + ostree_checksum_to_bytes (checksum), + g_variant_new_fixed_array (G_VARIANT_TYPE ("y"), + data, data_len, + 1)); + g_clear_object (&object_data_stream); + } + + if (!write_padding (pack_out, 4, pack_checksum, &offset, cancellable, error)) + goto out; + + /* offset points to aligned header size */ + index_entry = g_variant_new ("(u@ayt)", + GUINT32_TO_BE ((guint32)objtype), + ostree_checksum_to_bytes (checksum), + GUINT64_TO_BE (offset)); + g_ptr_array_add (index_content_list, g_variant_ref_sink (index_entry)); + + if (!write_variant_with_size (pack_out, packed_object, pack_checksum, + &offset, cancellable, error)) + goto out; + } + + if (!g_output_stream_close (pack_out, cancellable, error)) + goto out; + + g_variant_builder_init (&index_content_builder, G_VARIANT_TYPE ("a(uayt)")); + g_ptr_array_sort (index_content_list, compare_index_content); + for (i = 0; i < index_content_list->len; i++) + { + GVariant *index_item = index_content_list->pdata[i]; + g_variant_builder_add_value (&index_content_builder, index_item); + } + index_content = g_variant_new ("(s@a{sv}@a(uayt))", + "OSTv0PACKINDEX", + g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0), + g_variant_builder_end (&index_content_builder)); + + if (!g_output_stream_write_all (index_out, + g_variant_get_data (index_content), + g_variant_get_size (index_content), + &bytes_written, + cancellable, + error)) + goto out; + + if (!g_output_stream_close (index_out, cancellable, error)) + goto out; + + if (!ostree_repo_add_pack_file (data->repo, + g_checksum_get_string (pack_checksum), + index_temppath, + pack_temppath, + cancellable, + error)) + goto out; + + if (!ostree_repo_regenerate_pack_index (data->repo, cancellable, error)) + goto out; + + g_print ("Created pack file '%s' with %u objects\n", g_checksum_get_string (pack_checksum), objects->len); + + if (!opt_keep_loose) + { + for (i = 0; i < objects->len; i++) + { + GVariant *object_data = objects->pdata[i]; + const char *checksum; + guint32 objtype_u32; + OstreeObjectType objtype; + guint64 expected_objsize; + + g_variant_get (object_data, "(&sut)", &checksum, &objtype_u32, &expected_objsize); + + objtype = (OstreeObjectType) objtype_u32; + + if (!delete_loose_object (data, checksum, objtype, cancellable, error)) + goto out; + } + } + + ret = TRUE; + out: + if (index_temppath) + (void) unlink (ot_gfile_get_path_cached (index_temppath)); + g_clear_object (&index_temppath); + g_clear_object (&index_out); + if (pack_temppath) + (void) unlink (ot_gfile_get_path_cached (pack_temppath)); + g_clear_object (&pack_temppath); + g_clear_object (&pack_out); + g_clear_object (&object_path); + g_clear_object (&object_input); + g_clear_object (&compressor); + g_clear_object (&compressed_object_input); + g_clear_object (&object_file_info); + if (pack_checksum) + g_checksum_free (pack_checksum); + g_clear_object (&pack_dir); + ot_clear_gvariant (&index_content); + g_free (pack_name); + g_clear_object (&pack_file_path); + g_clear_object (&pack_index_path); + if (index_content_list) + g_ptr_array_unref (index_content_list); + return ret; +} + +/** + * cluster_objects_stupidly: + * @objects: Map from serialized object name to objdata + * @out_clusters: (out): [Array of [Array of object data]]. Free with g_ptr_array_unref(). + * + * Just sorts by size currently. Also filters out non-regular object + * content. + */ +static gboolean +cluster_objects_stupidly (OtRepackData *data, + GHashTable *objects, + GPtrArray **out_clusters, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GPtrArray *ret_clusters = NULL; + GPtrArray *object_list = NULL; + guint i; + guint64 current_size; + guint current_offset; + GHashTableIter hash_iter; + gpointer key, value; + GFile *object_path = NULL; + GFileInfo *object_info = NULL; + + object_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); + + g_hash_table_iter_init (&hash_iter, objects); + + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + GVariant *serialized_key = key; + const char *checksum; + OstreeObjectType objtype; + guint64 size; + + ostree_object_name_deserialize (serialized_key, &checksum, &objtype); + + g_clear_object (&object_path); + object_path = ostree_repo_get_object_path (data->repo, checksum, objtype); + + g_clear_object (&object_info); + object_info = g_file_query_info (object_path, OSTREE_GIO_FAST_QUERYINFO, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + cancellable, error); + if (!object_info) + goto out; + + if (g_file_info_get_file_type (object_info) != G_FILE_TYPE_REGULAR) + continue; + + size = g_file_info_get_attribute_uint64 (object_info, G_FILE_ATTRIBUTE_STANDARD_SIZE); + + g_ptr_array_add (object_list, + g_variant_ref_sink (g_variant_new ("(sut)", checksum, (guint32)objtype, size))); + } + + g_ptr_array_sort (object_list, compare_object_data_by_size); + + ret_clusters = g_ptr_array_new_with_free_func ((GDestroyNotify)g_ptr_array_unref); + + current_size = 0; + current_offset = 0; + for (i = 0; i < object_list->len; i++) + { + GVariant *objdata = object_list->pdata[i]; + guint64 objsize; + + g_variant_get_child (objdata, 2, "t", &objsize); + + if (current_size + objsize > data->pack_size || i == (object_list->len - 1)) + { + guint j; + GPtrArray *current; + + if (current_offset < i) + { + current = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); + for (j = current_offset; j <= i; j++) + { + g_ptr_array_add (current, g_variant_ref (object_list->pdata[j])); + } + g_ptr_array_add (ret_clusters, current); + current_size = objsize; + current_offset = i+1; + } + } + else if (objsize > data->pack_size) + { + break; + } + else + { + current_size += objsize; + } + } + + ret = TRUE; + ot_transfer_out_value (out_clusters, &ret_clusters); + out: + if (object_list) + g_ptr_array_unref (object_list); + return ret; +} + +static gboolean +parse_size_spec_with_suffix (const char *spec, + guint64 default_value, + guint64 *out_size, + GError **error) +{ + gboolean ret = FALSE; + char *endptr = NULL; + guint64 ret_size; + + if (spec == NULL) + { + ret_size = default_value; + endptr = NULL; + } + else + { + ret_size = g_ascii_strtoull (spec, &endptr, 10); + + if (endptr && *endptr) + { + char suffix = *endptr; + + switch (suffix) + { + case 'k': + case 'K': + { + ret_size *= 1024; + break; + } + case 'm': + case 'M': + { + ret_size *= (1024 * 1024); + break; + } + case 'g': + case 'G': + { + ret_size *= (1024 * 1024 * 1024); + break; + } + default: + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid size suffix '%c'", suffix); + goto out; + } + } + } + + ret = TRUE; + *out_size = ret_size; + out: + return ret; +} + +static gboolean +parse_compression_string (const char *compstr, + OtCompressionType *out_comptype, + GError **error) +{ + gboolean ret = FALSE; + OtCompressionType ret_comptype; + + if (compstr == NULL) + ret_comptype = OT_COMPRESSION_NONE; + else if (strcmp (compstr, "gzip") == 0) + ret_comptype = OT_COMPRESSION_GZIP; + else if (strcmp (compstr, "xz") == 0) + ret_comptype = OT_COMPRESSION_XZ; + else + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid compression '%s'", compstr); + goto out; + } + + ret = TRUE; + *out_comptype = ret_comptype; + out: + return ret; +} + +static gboolean +do_stats_gather_loose (OtRepackData *data, + GHashTable *objects, + GHashTable **out_loose, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GHashTable *ret_loose = NULL; + guint n_loose = 0; + guint n_loose_and_packed = 0; + guint n_packed = 0; + guint n_dup_packed = 0; + guint n_commits = 0; + guint n_dirmeta = 0; + guint n_dirtree = 0; + guint n_files = 0; + GHashTableIter hash_iter; + gpointer key, value; + + ret_loose = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, + (GDestroyNotify) g_variant_unref, + NULL); + + g_hash_table_iter_init (&hash_iter, objects); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + GVariant *serialized_key = key; + GVariant *objdata = value; + const char *checksum; + OstreeObjectType objtype; + gboolean is_loose; + gboolean is_packed; + GVariant *pack_array; + + ostree_object_name_deserialize (serialized_key, &checksum, &objtype); + + g_variant_get (objdata, "(b@as)", &is_loose, &pack_array); + + is_packed = g_variant_n_children (pack_array) > 0; + + if (is_loose && is_packed) + { + n_loose_and_packed++; + } + else if (is_loose) + { + GVariant *copy = g_variant_ref (serialized_key); + g_hash_table_replace (ret_loose, copy, copy); + n_loose++; + } + else if (g_variant_n_children (pack_array) > 1) + { + n_dup_packed++; + } + else + { + n_packed++; + } + + switch (objtype) + { + case OSTREE_OBJECT_TYPE_COMMIT: + n_commits++; + break; + case OSTREE_OBJECT_TYPE_DIR_TREE: + n_dirtree++; + break; + case OSTREE_OBJECT_TYPE_DIR_META: + n_dirmeta++; + break; + case OSTREE_OBJECT_TYPE_RAW_FILE: + case OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META: + n_files++; + break; + case OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT: + /* Counted under files by META */ + break; + } + } + + g_print ("Commits: %u\n", n_commits); + g_print ("Tree contents: %u\n", n_dirtree); + g_print ("Tree meta: %u\n", n_dirmeta); + g_print ("Files: %u\n", n_files); + g_print ("\n"); + g_print ("Loose+packed objects: %u\n", n_loose_and_packed); + g_print ("Loose-only objects: %u\n", n_loose); + g_print ("Duplicate packed objects: %u\n", n_dup_packed); + g_print ("Packed-only objects: %u\n", n_packed); + + ret = TRUE; + ot_transfer_out_value (out_loose, &ret_loose); + /* out: */ + if (ret_loose) + g_hash_table_unref (ret_loose); + return ret; +} + +static gboolean +do_incremental_pack (OtRepackData *data, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GHashTable *objects = NULL; + guint i; + GPtrArray *clusters = NULL; + GHashTable *loose_objects = NULL; + + if (!ostree_repo_list_objects (data->repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects, + cancellable, error)) + goto out; + + if (!do_stats_gather_loose (data, objects, &loose_objects, cancellable, error)) + goto out; + + g_print ("\n"); + g_print ("Using pack size: %" G_GUINT64_FORMAT "\n", data->pack_size); + + if (!cluster_objects_stupidly (data, loose_objects, &clusters, cancellable, error)) + goto out; + + if (clusters->len > 0) + g_print ("Going to create %u packfiles\n", clusters->len); + else + g_print ("Nothing to do\n"); + + for (i = 0; i < clusters->len; i++) + { + GPtrArray *cluster = clusters->pdata[i]; + + if (!opt_analyze_only) + { + if (!create_pack_file (data, cluster, cancellable, error)) + goto out; + } + } + + ret = TRUE; + out: + if (clusters) + g_ptr_array_unref (clusters); + if (loose_objects) + g_hash_table_unref (loose_objects); + if (objects) + g_hash_table_unref (objects); + return ret; +} + +gboolean +ostree_builtin_pack (int argc, char **argv, GFile *repo_path, GError **error) +{ + gboolean ret = FALSE; + GOptionContext *context; + OtRepackData data; + OstreeRepo *repo = NULL; + GCancellable *cancellable = NULL; + + memset (&data, 0, sizeof (data)); + + context = g_option_context_new ("- Recompress objects"); + g_option_context_add_main_entries (context, options, NULL); + + if (!g_option_context_parse (context, &argc, &argv, error)) + goto out; + + repo = ostree_repo_new (repo_path); + if (!ostree_repo_check (repo, error)) + goto out; + + if (ostree_repo_get_mode (repo) != OSTREE_REPO_MODE_ARCHIVE) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, + "Can't repack bare repositories yet"); + goto out; + } + + data.repo = repo; + data.error = error; + + if (!parse_size_spec_with_suffix (opt_pack_size, OT_DEFAULT_PACK_SIZE_BYTES, &data.pack_size, error)) + goto out; + /* Default internal compression to gzip */ + if (!parse_compression_string (opt_int_compression ? opt_int_compression : "gzip", &data.int_compression, error)) + goto out; + if (!parse_compression_string (opt_ext_compression, &data.ext_compression, error)) + goto out; + + if (opt_reindex_only) + { + if (!ostree_repo_regenerate_pack_index (repo, cancellable, error)) + goto out; + } + else + { + if (!do_incremental_pack (&data, cancellable, error)) + goto out; + } + + ret = TRUE; + out: + if (context) + g_option_context_free (context); + g_clear_object (&repo); + return ret; +} diff --git a/src/ostree/ot-builtin-prune.c b/src/ostree/ot-builtin-prune.c index 5090036b..2c50f515 100644 --- a/src/ostree/ot-builtin-prune.c +++ b/src/ostree/ot-builtin-prune.c @@ -187,24 +187,27 @@ compute_reachable_objects_from_commit (OstreeRepo *repo, return ret; } -static void -object_iter_callback (OstreeRepo *repo, - const char *checksum, - OstreeObjectType objtype, - GFile *objf, - GFileInfo *file_info, - gpointer user_data) +static gboolean +prune_loose_object (OtPruneData *data, + const char *checksum, + OstreeObjectType objtype, + GCancellable *cancellable, + GError **error) { - OtPruneData *data = user_data; + gboolean ret = FALSE; char *key; + GFile *objf = NULL; key = ostree_object_to_string (checksum, objtype); + objf = ostree_repo_get_object_path (data->repo, checksum, objtype); + if (!g_hash_table_lookup_extended (data->reachable, key, NULL, NULL)) { if (delete) { - (void) unlink (ot_gfile_get_path_cached (objf)); + if (!g_file_delete (objf, cancellable, error)) + goto out; g_print ("Deleted: %s\n", key); } else @@ -216,16 +219,21 @@ object_iter_callback (OstreeRepo *repo, else data->n_reachable++; + ret = TRUE; + out: + g_clear_object (&objf); g_free (key); + return ret; } gboolean ostree_builtin_prune (int argc, char **argv, GFile *repo_path, GError **error) { + gboolean ret = FALSE; GOptionContext *context; OtPruneData data; - gboolean ret = FALSE; + GHashTable *objects = NULL; OstreeRepo *repo = NULL; GHashTable *all_refs = NULL; GHashTableIter hash_iter; @@ -266,11 +274,37 @@ ostree_builtin_prune (int argc, char **argv, GFile *repo_path, GError **error) goto out; } - g_hash_table_iter_init (&hash_iter, data.reachable); - - if (!ostree_repo_iter_objects (repo, object_iter_callback, &data, error)) + if (!ostree_repo_list_objects (repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects, cancellable, error)) goto out; + g_hash_table_iter_init (&hash_iter, objects); + + + if (!ostree_repo_list_objects (repo, OSTREE_REPO_LIST_OBJECTS_ALL, + &objects, cancellable, error)) + goto out; + + g_hash_table_iter_init (&hash_iter, objects); + + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + GVariant *serialized_key = key; + GVariant *objdata = value; + const char *checksum; + OstreeObjectType objtype; + gboolean is_loose; + + ostree_object_name_deserialize (serialized_key, &checksum, &objtype); + + g_variant_get_child (objdata, 0, "b", &is_loose); + + if (is_loose) + { + if (!prune_loose_object (&data, checksum, objtype, cancellable, error)) + goto out; + } + } + if (data.had_error) goto out; @@ -286,5 +320,7 @@ ostree_builtin_prune (int argc, char **argv, GFile *repo_path, GError **error) if (context) g_option_context_free (context); g_clear_object (&repo); + if (objects) + g_hash_table_unref (objects); return ret; } diff --git a/src/ostree/ot-builtin-unpack.c b/src/ostree/ot-builtin-unpack.c new file mode 100644 index 00000000..eae4b04d --- /dev/null +++ b/src/ostree/ot-builtin-unpack.c @@ -0,0 +1,305 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2012 Colin Walters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Author: Colin Walters + */ + +#include "config.h" + +#include "ot-builtins.h" +#include "ostree.h" + +#include +#include + +#include +#include + +static GOptionEntry options[] = { + { NULL } +}; + +typedef struct { + OstreeRepo *repo; +} OtUnpackData; + +static gboolean +gather_packed (OtUnpackData *data, + GHashTable *objects, + GHashTable **out_packed, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GHashTable *ret_packed = NULL; + GHashTableIter hash_iter; + gpointer key, value; + GVariant *pack_array = NULL; + + ret_packed = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, + (GDestroyNotify) g_variant_unref, + NULL); + + g_hash_table_iter_init (&hash_iter, objects); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + GVariant *serialized_key = key; + GVariant *key_copy; + GVariant *objdata = value; + const char *checksum; + OstreeObjectType objtype; + gboolean is_loose; + gboolean is_packed; + + ostree_object_name_deserialize (serialized_key, &checksum, &objtype); + + ot_clear_gvariant (&pack_array); + g_variant_get (objdata, "(b@as)", &is_loose, &pack_array); + + is_packed = g_variant_n_children (pack_array) > 0; + + if (is_loose) + continue; + + g_assert (is_packed); + + key_copy = g_variant_ref (serialized_key); + g_hash_table_replace (ret_packed, key_copy, key_copy); + } + + ret = TRUE; + ot_transfer_out_value (out_packed, &ret_packed); + /* out: */ + ot_clear_gvariant (&pack_array); + if (ret_packed) + g_hash_table_unref (ret_packed); + return ret; +} + +static gboolean +unpack_one_object (OstreeRepo *repo, + const char *checksum, + OstreeObjectType objtype, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GInputStream *input = NULL; + GFileInfo *file_info = NULL; + GVariant *xattrs = NULL; + GVariant *meta = NULL; + GVariant *serialized_meta = NULL; + + g_assert (objtype != OSTREE_OBJECT_TYPE_RAW_FILE); + + if (objtype == OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META) + { + if (!ostree_repo_load_file (repo, checksum, + &input, &file_info, &xattrs, + cancellable, error)) + goto out; + + if (!ostree_repo_stage_object_trusted (repo, OSTREE_OBJECT_TYPE_RAW_FILE, + checksum, TRUE, file_info, xattrs, input, + cancellable, error)) + goto out; + } + else if (objtype == OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT) + { + /* nothing; handled in META case */ + } + else + { + if (!ostree_repo_load_variant (repo, objtype, checksum, &meta, error)) + goto out; + + serialized_meta = ostree_wrap_metadata_variant (objtype, meta); + + input = g_memory_input_stream_new_from_data (g_variant_get_data (serialized_meta), + g_variant_get_size (serialized_meta), NULL); + + if (!ostree_repo_stage_object_trusted (repo, objtype, checksum, TRUE, + NULL, NULL, input, cancellable, error)) + goto out; + } + + ret = TRUE; + out: + g_clear_object (&input); + g_clear_object (&file_info); + ot_clear_gvariant (&xattrs); + ot_clear_gvariant (&meta); + ot_clear_gvariant (&serialized_meta); + return ret; +} + +static gboolean +delete_one_packfile (OstreeRepo *repo, + const char *pack_checksum, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GFile *data_path = NULL; + GFile *index_path = NULL; + + index_path = ostree_repo_get_pack_index_path (repo, pack_checksum); + data_path = ostree_repo_get_pack_data_path (repo, pack_checksum); + + if (!ot_gfile_unlink (index_path, cancellable, error)) + { + g_prefix_error (error, "Failed to delete pack index '%s': ", ot_gfile_get_path_cached (index_path)); + goto out; + } + if (!ot_gfile_unlink (data_path, cancellable, error)) + { + g_prefix_error (error, "Failed to delete pack data '%s': ", ot_gfile_get_path_cached (data_path)); + goto out; + } + + ret = TRUE; + out: + g_clear_object (&index_path); + g_clear_object (&data_path); + return ret; +} + +gboolean +ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error) +{ + gboolean ret = FALSE; + GOptionContext *context; + gboolean in_transaction = FALSE; + OtUnpackData data; + OstreeRepo *repo = NULL; + GHashTable *objects = NULL; + GCancellable *cancellable = NULL; + GPtrArray *clusters = NULL; + GHashTable *packed_objects = NULL; + GHashTableIter hash_iter; + GHashTable *packfiles_to_delete = NULL; + gpointer key, value; + GFile *objpath = NULL; + guint64 unpacked_object_count = 0; + + memset (&data, 0, sizeof (data)); + + context = g_option_context_new ("- Uncompress objects"); + g_option_context_add_main_entries (context, options, NULL); + + if (!g_option_context_parse (context, &argc, &argv, error)) + goto out; + + repo = ostree_repo_new (repo_path); + if (!ostree_repo_check (repo, error)) + goto out; + + if (ostree_repo_get_mode (repo) != OSTREE_REPO_MODE_ARCHIVE) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, + "Can't unpack bare repositories yet"); + goto out; + } + + data.repo = repo; + + if (!ostree_repo_list_objects (repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects, cancellable, error)) + goto out; + + if (!gather_packed (&data, objects, &packed_objects, cancellable, error)) + goto out; + + if (!ostree_repo_prepare_transaction (repo, cancellable, error)) + goto out; + + in_transaction = TRUE; + + packfiles_to_delete = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); + + g_hash_table_iter_init (&hash_iter, packed_objects); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + GVariant *objkey = key; + GVariant *objdata; + const char *checksum; + const char *pack_checksum; + OstreeObjectType objtype; + gboolean is_loose; + GVariantIter *pack_array_iter; + + objdata = g_hash_table_lookup (objects, objkey); + g_assert (objdata); + + g_variant_get (objdata, "(bas)", &is_loose, &pack_array_iter); + + g_assert (!is_loose); + + while (g_variant_iter_loop (pack_array_iter, "&s", &pack_checksum)) + { + if (!g_hash_table_lookup (packfiles_to_delete, pack_checksum)) + { + gchar *duped_checksum = g_strdup (pack_checksum); + g_hash_table_replace (packfiles_to_delete, duped_checksum, duped_checksum); + } + } + g_variant_iter_free (pack_array_iter); + + ostree_object_name_deserialize (objkey, &checksum, &objtype); + + if (!unpack_one_object (repo, checksum, objtype, cancellable, error)) + goto out; + + unpacked_object_count++; + } + + if (!ostree_repo_commit_transaction (repo, cancellable, error)) + goto out; + + if (g_hash_table_size (packfiles_to_delete) == 0) + g_print ("No pack files; nothing to do\n"); + + g_hash_table_iter_init (&hash_iter, packfiles_to_delete); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + const char *pack_checksum = key; + + if (!delete_one_packfile (repo, pack_checksum, cancellable, error)) + goto out; + + g_print ("Deleted packfile '%s'\n", pack_checksum); + } + + ret = TRUE; + out: + if (in_transaction) + (void) ostree_repo_abort_transaction (repo, cancellable, NULL); + g_clear_object (&objpath); + if (context) + g_option_context_free (context); + g_clear_object (&repo); + if (clusters) + g_ptr_array_unref (clusters); + if (packfiles_to_delete) + g_hash_table_unref (packfiles_to_delete); + if (packed_objects) + g_hash_table_unref (packed_objects); + if (objects) + g_hash_table_unref (objects); + return ret; +} diff --git a/src/ostree/ot-builtins.h b/src/ostree/ot-builtins.h index befd2037..73afccc3 100644 --- a/src/ostree/ot-builtins.h +++ b/src/ostree/ot-builtins.h @@ -40,8 +40,10 @@ gboolean ostree_builtin_ls (int argc, char **argv, GFile *repo_path, GError **er gboolean ostree_builtin_prune (int argc, char **argv, GFile *repo_path, GError **error); gboolean ostree_builtin_fsck (int argc, char **argv, GFile *repo_path, GError **error); gboolean ostree_builtin_show (int argc, char **argv, GFile *repo_path, GError **error); +gboolean ostree_builtin_pack (int argc, char **argv, GFile *repo_path, GError **error); gboolean ostree_builtin_rev_parse (int argc, char **argv, GFile *repo_path, GError **error); gboolean ostree_builtin_remote (int argc, char **argv, GFile *repo_path, GError **error); +gboolean ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error); G_END_DECLS diff --git a/tests/t0001-archive.sh b/tests/t0001-archive.sh index 6bdc9aa5..47127326 100755 --- a/tests/t0001-archive.sh +++ b/tests/t0001-archive.sh @@ -21,7 +21,7 @@ set -e . libtest.sh -echo '1..10' +echo '1..19' setup_test_repository "archive" echo "ok setup" @@ -67,3 +67,36 @@ cd ${test_tmpdir} $OSTREE cat test2 /baz/cow > cow-contents assert_file_has_content cow-contents "moo" echo "ok cat-file" + +cd ${test_tmpdir} +$OSTREE pack --keep-loose +echo "ok pack" + +cd ${test_tmpdir} +$OSTREE fsck +echo "ok fsck" + +$OSTREE checkout test2 checkout-test2-from-packed +echo "ok checkout union 1" + +cd ${test_tmpdir} +$OSTREE pack +echo "ok pack delete loose" + +cd ${test_tmpdir} +$OSTREE fsck +echo "ok fsck" + +$OSTREE pack --analyze-only +echo "ok pack analyze" + +$OSTREE unpack +echo "ok unpack" + +cd ${test_tmpdir} +$OSTREE fsck +echo "ok fsck" + +cd ${test_tmpdir} +$OSTREE checkout test2 checkout-test2-from-unpacked +echo "ok checkout union 2" diff --git a/tests/t0010-pull.sh b/tests/t0010-pull.sh index 53c85b22..5f58d117 100755 --- a/tests/t0010-pull.sh +++ b/tests/t0010-pull.sh @@ -21,7 +21,7 @@ set -e . libtest.sh -echo '1..2' +echo '1..4' setup_fake_remote_repo1 cd ${test_tmpdir} @@ -29,6 +29,7 @@ mkdir repo ostree --repo=repo init ostree --repo=repo remote add origin $(cat httpd-address)/ostree/gnomerepo ostree-pull --repo=repo origin main +ostree --repo=repo fsck echo "ok pull" cd ${test_tmpdir} @@ -37,3 +38,21 @@ cd checkout-origin-main assert_file_has_content firstfile '^first$' assert_file_has_content baz/cow '^moo$' echo "ok pull contents" + +cd ${test_tmpdir} +ostree --repo=$(pwd)/ostree-srv/gnomerepo pack +rm -rf repo +mkdir repo +ostree --repo=repo init +ostree --repo=repo remote add origin $(cat httpd-address)/ostree/gnomerepo +ostree-pull --repo=repo origin main +ostree --repo=repo fsck +echo "ok pull packed" + +cd ${test_tmpdir} +rm -rf checkout-origin-main +$OSTREE checkout origin/main checkout-origin-main +cd checkout-origin-main +assert_file_has_content firstfile '^first$' +assert_file_has_content baz/cow '^moo$' +echo "ok pull contents packed"