packaging
/
ostree
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fetcher: Clean up code to avoid intermediate files 

I think originally we had the .part/.done separation because we were
trying to support partial downloads of files like repo/config and
repo/refs.

But now that the http server configuration won't give us partial
results, we don't need to support caching those files between runs.

And thus, there's no reason to have the .part/.done and do the dance
with renaming them.

When fetching objects/ and other things that use _with_async, we
continue to use _append_to(), and if the returned range tells us we
have all the bytes, then we hand the full file over to the caller.

Don't attempt to shortcut in the case where the last run told us we
already have the object; the object fetcher code will not make a
request.

While we're here, also clean up use of GError and consistently use the
cancellable from the pending.

https://bugzilla.gnome.org/show_bug.cgi?id=707157
This commit is contained in:
Colin Walters 2013-08-30 19:29:04 -04:00
parent 9e497a4ce7
commit 8b5f684b68
1 changed files with 74 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

238
src/libostree/ostree-fetcher.c
View File

@ -43,7 +43,6 @@ typedef struct {
SoupRequest *request; SoupRequest *request;
GFile *tmpfile; GFile *tmpfile;
gchar *filename; /* Hash of the SoupURI used to request the file */
GInputStream *request_body; GInputStream *request_body;
GOutputStream *out_stream; GOutputStream *out_stream;
@ -64,7 +63,6 @@ pending_uri_free (OstreeFetcherPendingURI *pending)
soup_uri_free (pending->uri); soup_uri_free (pending->uri);
g_clear_object (&pending->self); g_clear_object (&pending->self);
g_clear_object (&pending->tmpfile); g_clear_object (&pending->tmpfile);
g_free (pending->filename);
g_clear_object (&pending->request); g_clear_object (&pending->request);
g_clear_object (&pending->request_body); g_clear_object (&pending->request_body);
g_clear_object (&pending->out_stream); g_clear_object (&pending->out_stream);
@ -89,9 +87,6 @@ struct OstreeFetcher
guint total_requests; guint total_requests;
}; };
static const gchar *partsuffix = ".part";
static const gchar *donesuffix = ".done";
G_DEFINE_TYPE (OstreeFetcher, ostree_fetcher, G_TYPE_OBJECT) G_DEFINE_TYPE (OstreeFetcher, ostree_fetcher, G_TYPE_OBJECT)
static void static void
@ -172,37 +167,6 @@ ostree_fetcher_new (GFile *tmpdir,
return self; return self;
} }
static gboolean
rename_part_file (OstreeFetcherPendingURI *pending,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
GError *local_error = NULL;
const gchar *tempfilename = gs_file_get_path_cached (pending->tmpfile);
// Only rename files that end in .part
if (g_str_has_suffix (tempfilename, partsuffix))
{
gs_free gchar *finalname = g_strconcat (pending->filename, donesuffix, NULL);
gs_unref_object GFile *donefile = g_file_get_child (pending->self->tmpdir, finalname);
// Copy the .part file to .done file
if (!g_file_move (pending->tmpfile,
donefile,
G_FILE_COPY_OVERWRITE,
NULL,
NULL,
NULL,
&local_error))
goto out;
g_object_unref (pending->tmpfile);
pending->tmpfile = g_object_ref (donefile);
}
ret = TRUE;
out:
return ret;
}
static void static void
on_splice_complete (GObject *object, on_splice_complete (GObject *object,
GAsyncResult *result, GAsyncResult *result,
@ -210,30 +174,31 @@ on_splice_complete (GObject *object,
{ {
OstreeFetcherPendingURI *pending = user_data; OstreeFetcherPendingURI *pending = user_data;
gs_unref_object GFileInfo *file_info = NULL; gs_unref_object GFileInfo *file_info = NULL;
goffset filesize;
GError *local_error = NULL; GError *local_error = NULL;
pending->state = OSTREE_FETCHER_STATE_COMPLETE; pending->state = OSTREE_FETCHER_STATE_COMPLETE;
file_info = g_file_query_info (pending->tmpfile, OSTREE_GIO_FAST_QUERYINFO, file_info = g_file_query_info (pending->tmpfile, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
NULL, NULL); pending->cancellable, &local_error);
if (file_info) if (!file_info)
goto out;
filesize = g_file_info_get_size (file_info);
if (filesize < pending->content_length)
{ {
goffset filesize = g_file_info_get_size (file_info); g_set_error (&local_error, G_IO_ERROR, G_IO_ERROR_FAILED, "Download incomplete");
if (filesize < pending->content_length) goto out;
{ }
g_set_error (&local_error, G_IO_ERROR, G_IO_ERROR_FAILED, "Download incomplete"); else
g_simple_async_result_take_error (pending->result, local_error); {
} pending->self->total_downloaded += g_file_info_get_size (file_info);
else
{
pending->self->total_downloaded += g_file_info_get_size (file_info);
if (!rename_part_file (pending, NULL, &local_error))
g_simple_async_result_take_error (pending->result, local_error);
}
} }
out:
(void) g_input_stream_close (pending->request_body, NULL, NULL); (void) g_input_stream_close (pending->request_body, NULL, NULL);
if (local_error)
g_simple_async_result_take_error (pending->result, local_error);
g_simple_async_result_complete (pending->result); g_simple_async_result_complete (pending->result);
g_object_unref (pending->result); g_object_unref (pending->result);
} }
@ -266,11 +231,6 @@ on_request_sent (GObject *object,
{ {
// We already have the whole file, so just use it. // We already have the whole file, so just use it.
pending->state = OSTREE_FETCHER_STATE_COMPLETE; pending->state = OSTREE_FETCHER_STATE_COMPLETE;
if (!rename_part_file (pending, NULL, &local_error))
{
g_set_error (&local_error, G_IO_ERROR, G_IO_ERROR_FAILED, "Rename failed");
g_simple_async_result_take_error (pending->result, local_error);
}
g_simple_async_result_complete (pending->result); g_simple_async_result_complete (pending->result);
g_object_unref (pending->result); g_object_unref (pending->result);
return; return;
@ -305,37 +265,25 @@ on_request_sent (GObject *object,
} }
static OstreeFetcherPendingURI * static OstreeFetcherPendingURI *
ostree_fetcher_request_uri_internal (OstreeFetcher *self, ostree_fetcher_request_uri_internal (OstreeFetcher *self,
SoupURI *uri, SoupURI *uri,
GCancellable *cancellable) GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data,
gpointer source_tag)
{ {
OstreeFetcherPendingURI *pending; OstreeFetcherPendingURI *pending;
GError *local_error = NULL; GError *local_error = NULL;
gs_free char *uristring = soup_uri_to_string (uri, FALSE); gs_free char *uristring = soup_uri_to_string (uri, FALSE);
gs_free char *hash = g_compute_checksum_for_string (G_CHECKSUM_SHA256, uristring, strlen (uristring));
pending = g_new0 (OstreeFetcherPendingURI, 1); pending = g_new0 (OstreeFetcherPendingURI, 1);
pending->refcount = 1; pending->refcount = 1;
pending->self = g_object_ref (self); pending->self = g_object_ref (self);
pending->uri = soup_uri_copy (uri); pending->uri = soup_uri_copy (uri);
pending->filename = g_compute_checksum_for_string (G_CHECKSUM_SHA256, uristring, strlen (uristring)); pending->tmpfile = g_file_get_child (self->tmpdir, hash);
pending->cancellable = cancellable ? g_object_ref (cancellable) : NULL; pending->cancellable = cancellable ? g_object_ref (cancellable) : NULL;
pending->request = soup_requester_request_uri (self->requester, uri, &local_error); pending->request = soup_requester_request_uri (self->requester, uri, &local_error);
g_assert_no_error (local_error);
pending->refcount++;
return pending;
}
static void
ostree_fetcher_request_uri_use_existing_file (OstreeFetcher *self,
OstreeFetcherPendingURI *pending,
GAsyncReadyCallback callback,
gpointer user_data,
gpointer source_tag)
{
pending->state = OSTREE_FETCHER_STATE_COMPLETE;
pending->result = g_simple_async_result_new ((GObject*) self, pending->result = g_simple_async_result_new ((GObject*) self,
callback, callback,
user_data, user_data,
@ -343,8 +291,12 @@ ostree_fetcher_request_uri_use_existing_file (OstreeFetcher *self,
g_simple_async_result_set_op_res_gpointer (pending->result, g_simple_async_result_set_op_res_gpointer (pending->result,
pending, pending,
(GDestroyNotify) pending_uri_free); (GDestroyNotify) pending_uri_free);
g_simple_async_result_complete (pending->result);
g_object_unref (pending->result); g_assert_no_error (local_error);
pending->refcount++;
return pending;
} }
void void
@ -356,62 +308,36 @@ ostree_fetcher_request_uri_with_partial_async (OstreeFetcher *self,
{ {
OstreeFetcherPendingURI *pending; OstreeFetcherPendingURI *pending;
GError *local_error = NULL; GError *local_error = NULL;
gs_free char *finalname = NULL;
gs_free char *downloadname = NULL;
self->total_requests++; self->total_requests++;
pending = ostree_fetcher_request_uri_internal (self, uri, cancellable); pending = ostree_fetcher_request_uri_internal (self, uri, cancellable,
callback, user_data,
ostree_fetcher_request_uri_with_partial_async);
pending->out_stream = G_OUTPUT_STREAM (g_file_append_to (pending->tmpfile, G_FILE_CREATE_NONE, NULL, &local_error));
if (!pending->out_stream)
goto out;
finalname = g_strconcat (pending->filename, donesuffix, NULL); if (SOUP_IS_REQUEST_HTTP (pending->request))
downloadname = g_strconcat (pending->filename, partsuffix, NULL);
/* First check if the finalname file exists */
pending->tmpfile = g_file_get_child (pending->self->tmpdir, finalname);
if (g_file_query_exists (pending->tmpfile, NULL) )
{ {
// We already have the complete file, so just use it. SoupMessage *msg;
ostree_fetcher_request_uri_use_existing_file (self, gs_unref_object GFileInfo *file_info =
pending, g_file_query_info (pending->tmpfile, OSTREE_GIO_FAST_QUERYINFO,
callback, G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
user_data, NULL, &local_error);
ostree_fetcher_request_uri_with_partial_async); if (!file_info)
}
else
{
g_object_unref (pending->tmpfile);
pending->tmpfile = g_file_get_child (pending->self->tmpdir, downloadname);
pending->out_stream = G_OUTPUT_STREAM (g_file_append_to (pending->tmpfile, G_FILE_CREATE_NONE, NULL, &local_error));
if (!pending->out_stream)
goto out; goto out;
if (SOUP_IS_REQUEST_HTTP (pending->request)) msg = soup_request_http_get_message ((SoupRequestHTTP*) pending->request);
{ if (g_file_info_get_size (file_info) > 0)
SoupMessage *msg; soup_message_headers_set_range (msg->request_headers, g_file_info_get_size (file_info), -1);
gs_unref_object GFileInfo *file_info = g_hash_table_insert (self->message_to_request,
g_file_query_info (pending->tmpfile, OSTREE_GIO_FAST_QUERYINFO, soup_request_http_get_message ((SoupRequestHTTP*)pending->request),
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, pending);
NULL, &local_error);
if (!file_info)
goto out;
msg = soup_request_http_get_message ((SoupRequestHTTP*) pending->request);
if (g_file_info_get_size (file_info) > 0)
soup_message_headers_set_range (msg->request_headers, g_file_info_get_size (file_info), -1);
g_hash_table_insert (self->message_to_request,
soup_request_http_get_message ((SoupRequestHTTP*)pending->request),
pending);
}
pending->result = g_simple_async_result_new ((GObject*) self,
callback, user_data,
ostree_fetcher_request_uri_with_partial_async);
g_simple_async_result_set_op_res_gpointer (pending->result, pending,
(GDestroyNotify) pending_uri_free);
soup_request_send_async (pending->request, cancellable,
on_request_sent, pending);
} }
soup_request_send_async (pending->request, cancellable,
on_request_sent, pending);
out: out:
if (local_error != NULL) if (local_error != NULL)
@ -423,8 +349,8 @@ ostree_fetcher_request_uri_with_partial_async (OstreeFetcher *self,
GFile * GFile *
ostree_fetcher_request_uri_with_partial_finish (OstreeFetcher *self, ostree_fetcher_request_uri_with_partial_finish (OstreeFetcher *self,
GAsyncResult *result, GAsyncResult *result,
GError **error) GError **error)
{ {
GSimpleAsyncResult *simple; GSimpleAsyncResult *simple;
OstreeFetcherPendingURI *pending; OstreeFetcherPendingURI *pending;
@ -448,50 +374,34 @@ ostree_fetcher_request_uri_async (OstreeFetcher *self,
{ {
OstreeFetcherPendingURI *pending; OstreeFetcherPendingURI *pending;
GError *local_error = NULL; GError *local_error = NULL;
gs_free char *finalname = NULL;
self->total_requests++; self->total_requests++;
pending = ostree_fetcher_request_uri_internal (self, uri, cancellable); pending = ostree_fetcher_request_uri_internal (self, uri, cancellable,
callback, user_data,
ostree_fetcher_request_uri_async);
finalname = g_strconcat (pending->filename, donesuffix, NULL); pending->out_stream = G_OUTPUT_STREAM (g_file_replace (pending->tmpfile, NULL, FALSE,
G_FILE_CREATE_REPLACE_DESTINATION,
cancellable, &local_error));
if (!pending->out_stream)
goto out;
/* TODO - make this async */ if (SOUP_IS_REQUEST_HTTP (pending->request))
pending->tmpfile = g_file_get_child (pending->self->tmpdir, finalname);
if (g_file_query_exists (pending->tmpfile, NULL) )
{ {
// We already have the complete file, so just use it. g_hash_table_insert (self->message_to_request,
ostree_fetcher_request_uri_use_existing_file (self, soup_request_http_get_message ((SoupRequestHTTP*)pending->request),
pending, pending);
callback,
user_data,
ostree_fetcher_request_uri_async);
} }
else
soup_request_send_async (pending->request, cancellable,
on_request_sent, pending);
out:
if (local_error)
{ {
pending->out_stream = G_OUTPUT_STREAM (g_file_append_to (pending->tmpfile, G_FILE_CREATE_NONE, NULL, &local_error)); g_simple_async_result_take_error (pending->result, local_error);
if (local_error) g_simple_async_result_complete (pending->result);
{
g_simple_async_result_take_error (pending->result, local_error);
g_simple_async_result_complete (pending->result);
return;
}
if (SOUP_IS_REQUEST_HTTP (pending->request))
{
g_hash_table_insert (self->message_to_request,
soup_request_http_get_message ((SoupRequestHTTP*)pending->request),
pending);
}
pending->result = g_simple_async_result_new ((GObject*) self,
callback, user_data,
ostree_fetcher_request_uri_async);
g_simple_async_result_set_op_res_gpointer (pending->result, pending,
(GDestroyNotify) pending_uri_free);
soup_request_send_async (pending->request, cancellable,
on_request_sent, pending);
} }
} }