repo: Allocate a tmpdir for each OstreeFetcher to isolate concurrent downloads

This way two pulls will not use the same tmpdir and accidentally
overwrite each other. However, consecutive OstreeFetchers will reuse
the tmpdirs, so that we can properly resume downloading large objects.

https://bugzilla.gnome.org/show_bug.cgi?id=757611

src/libostree/ostree-fetcher.c

@@ -30,6 +30,7 @@
 #include "ostree-tls-cert-interaction.h"
 #endif
 #include "ostree.h"
+#include "ostree-repo-private.h"
 #include "otutil.h"
 
 typedef enum {
@@ -63,6 +64,9 @@ struct OstreeFetcher
   GObject parent_instance;
 
   int tmpdir_dfd;
+  char *tmpdir_name;
+  GLnxLockFile tmpdir_lock;
+  int base_tmpdir_dfd;
 
   GTlsCertificate *client_cert;
 
@@ -114,6 +118,17 @@ _ostree_fetcher_finalize (GObject *object)
 
   self = OSTREE_FETCHER (object);
 
+  if (self->tmpdir_dfd != -1)
+    close (self->tmpdir_dfd);
+
+  /* Note: We don't remove the tmpdir here, because that would cause
+     us to not reuse it on resume. This happens because we use two
+     fetchers for each pull, so finalizing the first one would remove
+     all the files to be resumed from the previous second one */
+
+  g_free (self->tmpdir_name);
+  glnx_release_lock_file (&self->tmpdir_lock);
+
   g_clear_object (&self->session);
   g_clear_object (&self->client_cert);
 
@@ -140,6 +155,7 @@ _ostree_fetcher_init (OstreeFetcher *self)
 {
   gint max_conns;
   const char *http_proxy;
+  GLnxLockFile empty_lockfile = GLNX_LOCK_FILE_INIT;
 
   g_queue_init (&self->pending_queue);
   self->session = soup_session_async_new_with_options (SOUP_SESSION_USER_AGENT, "ostree ",
@@ -174,18 +190,33 @@ _ostree_fetcher_init (OstreeFetcher *self)
   self->output_stream_set = g_hash_table_new_full (NULL, NULL, NULL, (GDestroyNotify)g_object_unref);
 
   self->outstanding = g_hash_table_new_full (NULL, NULL, NULL, NULL);
+
+  self->tmpdir_dfd = -1;
+  self->tmpdir_lock = empty_lockfile;
+
 }
 
 OstreeFetcher *
 _ostree_fetcher_new (int                      tmpdir_dfd,
-                    OstreeFetcherConfigFlags  flags)
+                     OstreeFetcherConfigFlags flags,
+                     GCancellable            *cancellable,
+                     GError                 **error)
 {
   OstreeFetcher *self = (OstreeFetcher*)g_object_new (OSTREE_TYPE_FETCHER, NULL);
 
-  self->tmpdir_dfd = tmpdir_dfd;
+  if (!_ostree_repo_allocate_tmpdir (tmpdir_dfd,
+                                     "fetcher-",
+                                     &self->tmpdir_name,
+                                     &self->tmpdir_dfd,
+                                     &self->tmpdir_lock,
+                                     NULL,
+                                     cancellable, error))
+    return NULL;
+
+  self->base_tmpdir_dfd = tmpdir_dfd;
   if ((flags & OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE) > 0)
     g_object_set ((GObject*)self->session, "ssl-strict", FALSE, NULL);
- 
+
   return self;
 }
 

src/libostree/ostree-fetcher.h

@@ -54,8 +54,10 @@ typedef enum {
 
 GType   _ostree_fetcher_get_type (void) G_GNUC_CONST;
 
-OstreeFetcher *_ostree_fetcher_new (int                     tmpdir_dfd,
+OstreeFetcher *_ostree_fetcher_new (int                      tmpdir_dfd,
-                                    OstreeFetcherConfigFlags   flags);
+                                    OstreeFetcherConfigFlags flags,
+                                    GCancellable            *cancellable,
+                                    GError                 **error);
 
 int  _ostree_fetcher_get_dfd (OstreeFetcher *fetcher);
 

src/libostree/ostree-repo-pull.c

@@ -646,6 +646,7 @@ content_fetch_on_complete (GObject        *object,
                            GAsyncResult   *result,
                            gpointer        user_data) 
 {
+  OstreeFetcher *fetcher = (OstreeFetcher *)object;
   FetchObjectData *fetch_data = user_data;
   OtPullData *pull_data = fetch_data->pull_data;
   GError *local_error = NULL;
@@ -660,7 +661,7 @@ content_fetch_on_complete (GObject        *object,
   const char *checksum;
   OstreeObjectType objtype;
 
-  temp_path = _ostree_fetcher_request_uri_with_partial_finish ((OstreeFetcher*)object, result, error);
+  temp_path = _ostree_fetcher_request_uri_with_partial_finish (fetcher, result, error);
   if (!temp_path)
     goto out;
 
@@ -680,7 +681,7 @@ content_fetch_on_complete (GObject        *object,
       if (!have_object)
         {
           if (!_ostree_repo_commit_loose_final (pull_data->repo, checksum, OSTREE_OBJECT_TYPE_FILE,
-                                                pull_data->tmpdir_dfd, temp_path,
+                                                _ostree_fetcher_get_dfd (fetcher), temp_path,
                                                 cancellable, error))
             goto out;
         }
@@ -689,13 +690,14 @@ content_fetch_on_complete (GObject        *object,
   else
     {
       /* Non-mirroring path */
-      
+
-      if (!ostree_content_file_parse_at (TRUE, pull_data->tmpdir_dfd, temp_path, FALSE,
+      if (!ostree_content_file_parse_at (TRUE, _ostree_fetcher_get_dfd (fetcher),
+                                         temp_path, FALSE,
                                          &file_in, &file_info, &xattrs,
                                          cancellable, error))
         {
           /* If it appears corrupted, delete it */
-          (void) unlinkat (pull_data->tmpdir_dfd, temp_path, 0);
+          (void) unlinkat (_ostree_fetcher_get_dfd (fetcher), temp_path, 0);
           goto out;
         }
 
@@ -703,8 +705,8 @@ content_fetch_on_complete (GObject        *object,
        * a reference to the fd.  If we fail to write later, then
        * the temp space will be cleaned up.
        */
-      (void) unlinkat (pull_data->tmpdir_dfd, temp_path, 0);
+      (void) unlinkat (_ostree_fetcher_get_dfd (fetcher), temp_path, 0);
-      
+
       if (!ostree_raw_file_to_content_stream (file_in, file_info, xattrs,
                                               &object_input, &length,
                                               cancellable, error))
@@ -772,6 +774,7 @@ meta_fetch_on_complete (GObject           *object,
                         GAsyncResult      *result,
                         gpointer           user_data)
 {
+  OstreeFetcher *fetcher = (OstreeFetcher *)object;
   FetchObjectData *fetch_data = user_data;
   OtPullData *pull_data = fetch_data->pull_data;
   g_autoptr(GVariant) metadata = NULL;
@@ -786,7 +789,7 @@ meta_fetch_on_complete (GObject           *object,
   g_debug ("fetch of %s%s complete", ostree_object_to_string (checksum, objtype),
            fetch_data->is_detached_meta ? " (detached)" : "");
 
-  temp_path = _ostree_fetcher_request_uri_with_partial_finish ((OstreeFetcher*)object, result, error);
+  temp_path = _ostree_fetcher_request_uri_with_partial_finish (fetcher, result, error);
   if (!temp_path)
     {
       if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
@@ -823,7 +826,7 @@ meta_fetch_on_complete (GObject           *object,
   if (objtype == OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT)
     goto out;
 
-  fd = openat (pull_data->tmpdir_dfd, temp_path, O_RDONLY | O_CLOEXEC);
+  fd = openat (_ostree_fetcher_get_dfd (fetcher), temp_path, O_RDONLY | O_CLOEXEC);
   if (fd == -1)
     {
       gs_set_error_from_errno (error, errno);
@@ -837,7 +840,7 @@ meta_fetch_on_complete (GObject           *object,
         goto out;
 
       /* Now delete it, see comment in corresponding content fetch path */
-      (void) unlinkat (pull_data->tmpdir_dfd, temp_path, 0);
+      (void) unlinkat (_ostree_fetcher_get_dfd (fetcher), temp_path, 0);
 
       if (!ostree_repo_write_commit_detached_metadata (pull_data->repo, checksum, metadata,
                                                        pull_data->cancellable, error))
@@ -852,7 +855,7 @@ meta_fetch_on_complete (GObject           *object,
                                    FALSE, &metadata, error))
         goto out;
 
-      (void) unlinkat (pull_data->tmpdir_dfd, temp_path, 0);
+      (void) unlinkat (_ostree_fetcher_get_dfd (fetcher), temp_path, 0);
 
       /* Write the commitpartial file now while we're still fetching data */
       if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
@@ -926,6 +929,7 @@ static_deltapart_fetch_on_complete (GObject           *object,
                                     GAsyncResult      *result,
                                     gpointer           user_data)
 {
+  OstreeFetcher *fetcher = (OstreeFetcher *)object;
   FetchStaticDeltaData *fetch_data = user_data;
   OtPullData *pull_data = fetch_data->pull_data;
   g_autoptr(GVariant) metadata = NULL;
@@ -939,11 +943,11 @@ static_deltapart_fetch_on_complete (GObject           *object,
 
   g_debug ("fetch static delta part %s complete", fetch_data->expected_checksum);
 
-  temp_path = _ostree_fetcher_request_uri_with_partial_finish ((OstreeFetcher*)object, result, error);
+  temp_path = _ostree_fetcher_request_uri_with_partial_finish (fetcher, result, error);
   if (!temp_path)
     goto out;
 
-  fd = openat (pull_data->tmpdir_dfd, temp_path, O_RDONLY | O_CLOEXEC);
+  fd = openat (_ostree_fetcher_get_dfd (fetcher), temp_path, O_RDONLY | O_CLOEXEC);
   if (fd == -1)
     {
       gs_set_error_from_errno (error, errno);
@@ -982,7 +986,7 @@ static_deltapart_fetch_on_complete (GObject           *object,
      * or error, the file will be gone.  This is particularly
      * important if say we hit e.g. ENOSPC.
      */
-    (void) unlinkat (pull_data->tmpdir_dfd, temp_path, 0); 
+    (void) unlinkat (_ostree_fetcher_get_dfd (fetcher), temp_path, 0);
 
     _ostree_static_delta_part_execute_async (pull_data->repo,
                                              fetch_data->objects,

src/libostree/ostree-repo.c

@@ -419,7 +419,9 @@ _ostree_repo_remote_new_fetcher (OstreeRepo  *self,
   if (tls_permissive)
     fetcher_flags |= OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE;
 
-  fetcher = _ostree_fetcher_new (self->tmp_dir_fd, fetcher_flags);
+  fetcher = _ostree_fetcher_new (self->tmp_dir_fd, fetcher_flags, NULL, error);
+  if (fetcher == NULL)
+    goto out;
 
   {
     g_autofree char *tls_client_cert_path = NULL;