packaging
/
ostree
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

repo-pull: Add a queue for scanning 

On systems with slow disks, the recursive scanning of directories can
be expensive -- it takes upwards of 2 minutes on our systems. This can
block the main loop for such a long time that it allows the download to
time out...

As such, move all the scanning of objects to a queue, processed from
an idle, to make sure that we don't block the main loop when scanning.

https://bugzilla.gnome.org/show_bug.cgi?id=753336
This commit is contained in:
Jasper St. Pierre 2015-08-06 15:45:21 -07:00
parent 863f5d8598
commit 20647edcbf
1 changed files with 102 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

160
src/libostree/ostree-repo-pull.c
View File

@ -98,6 +98,9 @@ typedef struct {
GError **async_error; GError **async_error;
gboolean caught_error; gboolean caught_error;
GQueue scan_object_queue;
GSource *idle_src;
} OtPullData; } OtPullData;
typedef struct { typedef struct {
@ -117,17 +120,26 @@ typedef struct {
char *expected_checksum; char *expected_checksum;
} FetchStaticDeltaData; } FetchStaticDeltaData;
typedef struct {
guchar csum[32];
OstreeObjectType objtype;
guint recursion_depth;
} ScanObjectQueueData;
static SoupURI * static SoupURI *
suburi_new (SoupURI *base, suburi_new (SoupURI *base,
const char *first, const char *first,
...) G_GNUC_NULL_TERMINATED; ...) G_GNUC_NULL_TERMINATED;
static gboolean scan_one_metadata_object (OtPullData *pull_data, static void queue_scan_one_metadata_object (OtPullData *pull_data,
const char *csum, const char *csum,
OstreeObjectType objtype, OstreeObjectType objtype,
guint recursion_depth, guint recursion_depth);
GCancellable *cancellable,
GError **error); static void queue_scan_one_metadata_object_c (OtPullData *pull_data,
const guchar *csum,
OstreeObjectType objtype,
guint recursion_depth);
static gboolean scan_one_metadata_object_c (OtPullData *pull_data, static gboolean scan_one_metadata_object_c (OtPullData *pull_data,
const guchar *csum, const guchar *csum,
@ -242,7 +254,8 @@ pull_termination_condition (OtPullData *pull_data)
gboolean current_write_idle = (pull_data->n_outstanding_metadata_write_requests == 0 && gboolean current_write_idle = (pull_data->n_outstanding_metadata_write_requests == 0 &&
pull_data->n_outstanding_content_write_requests == 0 && pull_data->n_outstanding_content_write_requests == 0 &&
pull_data->n_outstanding_deltapart_write_requests == 0 ); pull_data->n_outstanding_deltapart_write_requests == 0 );
gboolean current_idle = current_fetch_idle && current_write_idle; gboolean current_scan_idle = g_queue_is_empty (&pull_data->scan_object_queue);
gboolean current_idle = current_fetch_idle && current_write_idle && current_scan_idle;
if (pull_data->caught_error) if (pull_data->caught_error)
return TRUE; return TRUE;
@ -282,6 +295,47 @@ check_outstanding_requests_handle_error (OtPullData *pull_data,
} }
} }
static gboolean
idle_worker (gpointer user_data)
{
OtPullData *pull_data = user_data;
ScanObjectQueueData *scan_data;
GError *error = NULL;
scan_data = g_queue_pop_head (&pull_data->scan_object_queue);
if (!scan_data)
{
g_clear_pointer (&pull_data->idle_src, (GDestroyNotify) g_source_destroy);
return G_SOURCE_REMOVE;
}
scan_one_metadata_object_c (pull_data,
scan_data->csum,
scan_data->objtype,
scan_data->recursion_depth,
pull_data->cancellable,
&error);
check_outstanding_requests_handle_error (pull_data, error);
g_free (scan_data);
return G_SOURCE_CONTINUE;
}
static void
ensure_idle_queued (OtPullData *pull_data)
{
GSource *idle_src;
if (pull_data->idle_src)
return;
idle_src = g_idle_source_new ();
g_source_set_callback (idle_src, idle_worker, pull_data, NULL);
g_source_attach (idle_src, pull_data->main_context);
g_source_unref (idle_src);
pull_data->idle_src = idle_src;
}
typedef struct { typedef struct {
OtPullData *pull_data; OtPullData *pull_data;
GInputStream *result_stream; GInputStream *result_stream;
@ -454,16 +508,11 @@ scan_dirtree_object (OtPullData *pull_data,
if (subdir_target && strcmp (subdir_target, dirname) != 0) if (subdir_target && strcmp (subdir_target, dirname) != 0)
continue; continue;
if (!scan_one_metadata_object_c (pull_data, ostree_checksum_bytes_peek (tree_csum), queue_scan_one_metadata_object_c (pull_data, ostree_checksum_bytes_peek (tree_csum),
OSTREE_OBJECT_TYPE_DIR_TREE, recursion_depth + 1, OSTREE_OBJECT_TYPE_DIR_TREE, recursion_depth + 1);
cancellable, error)) queue_scan_one_metadata_object_c (pull_data, ostree_checksum_bytes_peek (meta_csum),
goto out; OSTREE_OBJECT_TYPE_DIR_META, recursion_depth + 1);
if (!scan_one_metadata_object_c (pull_data, ostree_checksum_bytes_peek (meta_csum),
OSTREE_OBJECT_TYPE_DIR_META, recursion_depth + 1,
cancellable, error))
goto out;
} }
ret = TRUE; ret = TRUE;
@ -696,9 +745,7 @@ on_metadata_written (GObject *object,
goto out; goto out;
} }
if (!scan_one_metadata_object_c (pull_data, csum, objtype, 0, queue_scan_one_metadata_object_c (pull_data, csum, objtype, 0);
pull_data->cancellable, error))
goto out;
out: out:
pull_data->n_outstanding_metadata_write_requests--; pull_data->n_outstanding_metadata_write_requests--;
@ -994,11 +1041,8 @@ scan_commit_object (OtPullData *pull_data,
have_parent = g_variant_n_children (parent_csum) > 0; have_parent = g_variant_n_children (parent_csum) > 0;
if (have_parent && pull_data->maxdepth == -1) if (have_parent && pull_data->maxdepth == -1)
{ {
if (!scan_one_metadata_object_c (pull_data, queue_scan_one_metadata_object_c (pull_data, ostree_checksum_bytes_peek (parent_csum),
ostree_checksum_bytes_peek (parent_csum), OSTREE_OBJECT_TYPE_COMMIT, recursion_depth + 1);
OSTREE_OBJECT_TYPE_COMMIT, recursion_depth + 1,
cancellable, error))
goto out;
} }
else if (have_parent && depth > 0) else if (have_parent && depth > 0)
{ {
@ -1022,48 +1066,49 @@ scan_commit_object (OtPullData *pull_data,
{ {
g_hash_table_insert (pull_data->commit_to_depth, g_strdup (parent_checksum), g_hash_table_insert (pull_data->commit_to_depth, g_strdup (parent_checksum),
GINT_TO_POINTER (parent_depth)); GINT_TO_POINTER (parent_depth));
if (!scan_one_metadata_object_c (pull_data, queue_scan_one_metadata_object_c (pull_data, ostree_checksum_bytes_peek (parent_csum),
ostree_checksum_bytes_peek (parent_csum), OSTREE_OBJECT_TYPE_COMMIT, recursion_depth + 1);
OSTREE_OBJECT_TYPE_COMMIT, recursion_depth + 1,
cancellable, error))
goto out;
} }
} }
g_variant_get_child (commit, 6, "@ay", &tree_contents_csum); g_variant_get_child (commit, 6, "@ay", &tree_contents_csum);
g_variant_get_child (commit, 7, "@ay", &tree_meta_csum); g_variant_get_child (commit, 7, "@ay", &tree_meta_csum);
if (!scan_one_metadata_object_c (pull_data, queue_scan_one_metadata_object_c (pull_data, ostree_checksum_bytes_peek (tree_contents_csum),
ostree_checksum_bytes_peek (tree_contents_csum), OSTREE_OBJECT_TYPE_DIR_TREE, recursion_depth + 1);
OSTREE_OBJECT_TYPE_DIR_TREE, recursion_depth + 1, queue_scan_one_metadata_object_c (pull_data, ostree_checksum_bytes_peek (tree_meta_csum),
cancellable, error)) OSTREE_OBJECT_TYPE_DIR_META, recursion_depth + 1);
goto out;
if (!scan_one_metadata_object_c (pull_data,
ostree_checksum_bytes_peek (tree_meta_csum),
OSTREE_OBJECT_TYPE_DIR_META, recursion_depth + 1,
cancellable, error))
goto out;
ret = TRUE; ret = TRUE;
out: out:
return ret; return ret;
} }
static gboolean static void
scan_one_metadata_object (OtPullData *pull_data, queue_scan_one_metadata_object (OtPullData *pull_data,
const char *csum, const char *csum,
OstreeObjectType objtype, OstreeObjectType objtype,
guint recursion_depth, guint recursion_depth)
GCancellable *cancellable,
GError **error)
{ {
guchar buf[32]; guchar buf[32];
ostree_checksum_inplace_to_bytes (csum, buf); ostree_checksum_inplace_to_bytes (csum, buf);
queue_scan_one_metadata_object_c (pull_data, buf, objtype, recursion_depth);
return scan_one_metadata_object_c (pull_data, buf, objtype, }
recursion_depth,
cancellable, error); static void
queue_scan_one_metadata_object_c (OtPullData *pull_data,
const guchar *csum,
OstreeObjectType objtype,
guint recursion_depth)
{
ScanObjectQueueData *scan_data = g_new0 (ScanObjectQueueData, 1);
memcpy (scan_data->csum, csum, sizeof (scan_data->csum));
scan_data->objtype = objtype;
scan_data->recursion_depth = recursion_depth;
g_queue_push_tail (&pull_data->scan_object_queue, scan_data);
ensure_idle_queued (pull_data);
} }
static gboolean static gboolean
@ -1675,6 +1720,7 @@ ostree_repo_pull_with_options (OstreeRepo *self,
pull_data->requested_metadata = g_hash_table_new_full (g_str_hash, g_str_equal, pull_data->requested_metadata = g_hash_table_new_full (g_str_hash, g_str_equal,
(GDestroyNotify)g_free, NULL); (GDestroyNotify)g_free, NULL);
pull_data->dir = g_strdup (dir_to_pull); pull_data->dir = g_strdup (dir_to_pull);
g_queue_init (&pull_data->scan_object_queue);
pull_data->start_time = g_get_monotonic_time (); pull_data->start_time = g_get_monotonic_time ();
@ -2021,9 +2067,7 @@ ostree_repo_pull_with_options (OstreeRepo *self,
while (g_hash_table_iter_next (&hash_iter, &key, &value)) while (g_hash_table_iter_next (&hash_iter, &key, &value))
{ {
const char *commit = value; const char *commit = value;
if (!scan_one_metadata_object (pull_data, commit, OSTREE_OBJECT_TYPE_COMMIT, queue_scan_one_metadata_object (pull_data, commit, OSTREE_OBJECT_TYPE_COMMIT, 0);
0, pull_data->cancellable, error))
goto out;
} }
g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch); g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch);
@ -2050,9 +2094,7 @@ ostree_repo_pull_with_options (OstreeRepo *self,
if (!delta_superblock) if (!delta_superblock)
{ {
g_debug ("no delta superblock for %s-%s", from_revision ? from_revision : "empty", to_revision); g_debug ("no delta superblock for %s-%s", from_revision ? from_revision : "empty", to_revision);
if (!scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, 0);
0, pull_data->cancellable, error))
goto out;
} }
else else
{ {
@ -2077,6 +2119,7 @@ ostree_repo_pull_with_options (OstreeRepo *self,
/* Now await work completion */ /* Now await work completion */
while (!pull_termination_condition (pull_data)) while (!pull_termination_condition (pull_data))
g_main_context_iteration (pull_data->main_context, TRUE); g_main_context_iteration (pull_data->main_context, TRUE);
if (pull_data->caught_error) if (pull_data->caught_error)
goto out; goto out;
@ -2203,6 +2246,7 @@ ostree_repo_pull_with_options (OstreeRepo *self,
g_clear_pointer (&pull_data->summary_deltas_checksums, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->summary_deltas_checksums, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->requested_content, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->requested_content, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->requested_metadata, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->requested_metadata, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->idle_src, (GDestroyNotify) g_source_destroy);
g_clear_pointer (&remote_config, (GDestroyNotify) g_key_file_unref); g_clear_pointer (&remote_config, (GDestroyNotify) g_key_file_unref);
return ret; return ret;
} }