fetcher queue: also throttle on outstanding writes

When fetching over a fast enough connection, we can be receiving files
faster than we write them. This can then lead to EMFILE when we have
enough files open. This was made very easy to notice with the upcoming
libcurl backend, which makes use of pipelining.

Closes: #675
Approved by: cgwalters

src/libostree/ostree-repo-private.h

@@ -35,6 +35,14 @@ G_BEGIN_DECLS
 #define _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS 8
 #define _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS 2
 
+/* In most cases, writing to disk should be much faster than
+ * fetching from the network, so we shouldn't actually hit
+ * this. But if using pipelining and e.g. pulling over LAN
+ * (or writing to slow media), we can have a runaway
+ * situation towards EMFILE.
+ * */
+#define _OSTREE_MAX_OUTSTANDING_WRITE_REQUESTS 16
+
 typedef enum {
   OSTREE_REPO_TEST_ERROR_PRE_COMMIT = (1 << 0)
 } OstreeRepoTestErrorFlags;

src/libostree/ostree-repo-pull.c

@@ -345,15 +345,26 @@ check_outstanding_requests_handle_error (OtPullData          *pull_data,
 
 /* We have a total-request limit, as well has a hardcoded max of 2 for delta
  * parts. The logic for the delta one is that processing them is expensive, and
- * doing multiple simultaneously could risk space/memory on smaller devices.
+ * doing multiple simultaneously could risk space/memory on smaller devices. We
+ * also throttle on outstanding writes in case fetches are faster.
  */
 static gboolean
 fetcher_queue_is_full (OtPullData *pull_data)
 {
-  return (pull_data->n_outstanding_metadata_fetches +
-          pull_data->n_outstanding_content_fetches +
-          pull_data->n_outstanding_deltapart_fetches) == _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS ||
-    pull_data->n_outstanding_deltapart_fetches == _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS;
+  const gboolean fetch_full =
+      ((pull_data->n_outstanding_metadata_fetches +
+        pull_data->n_outstanding_content_fetches +
+        pull_data->n_outstanding_deltapart_fetches) ==
+         _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS);
+  const gboolean deltas_full =
+      (pull_data->n_outstanding_deltapart_fetches ==
+        _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS);
+  const gboolean writes_full =
+      ((pull_data->n_outstanding_metadata_write_requests +
+        pull_data->n_outstanding_content_write_requests +
+        pull_data->n_outstanding_deltapart_write_requests) >=
+         _OSTREE_MAX_OUTSTANDING_WRITE_REQUESTS);
+  return fetch_full || deltas_full || writes_full;
 }
 
 static gboolean