lib/fetcher: Allow clients to append to User-Agent

We do already have `http-headers`, which potentially could be used to
allow clients to completely override the field, but it seems like the
more common use case is simply to append.

Closes: #1496
Approved by: cgwalters

src/libostree/ostree-fetcher-curl.c

@@ -77,6 +77,7 @@ struct OstreeFetcher
   char *proxy;
   struct curl_slist *extra_headers;
   int tmpdir_dfd;
+  char *custom_user_agent;
 
   GMainContext *mainctx;
   CURLM *multi;
@@ -180,6 +181,7 @@ _ostree_fetcher_finalize (GObject *object)
   g_clear_pointer (&self->timer_event, (GDestroyNotify)destroy_and_unref_source);
   if (self->mainctx)
     g_main_context_unref (self->mainctx);
+  g_clear_pointer (&self->custom_user_agent, (GDestroyNotify)g_free);
 
   G_OBJECT_CLASS (_ostree_fetcher_parent_class)->finalize (object);
 }
@@ -676,6 +678,18 @@ _ostree_fetcher_set_extra_headers (OstreeFetcher *self,
     }
 }
 
+void
+_ostree_fetcher_set_extra_user_agent (OstreeFetcher *self,
+                                      const char    *extra_user_agent)
+{
+  g_clear_pointer (&self->custom_user_agent, (GDestroyNotify)g_free);
+  if (extra_user_agent)
+    {
+      self->custom_user_agent =
+        g_strdup_printf ("%s %s", OSTREE_FETCHER_USERAGENT_STRING, extra_user_agent);
+    }
+}
+
 /* Re-bind all of the outstanding curl items to our new main context */
 static void
 adopt_steal_mainctx (OstreeFetcher *self,
@@ -716,7 +730,8 @@ initiate_next_curl_request (FetcherRequest *req,
     curl_easy_setopt (req->easy, CURLOPT_URL, uri);
   }
 
-  curl_easy_setopt (req->easy, CURLOPT_USERAGENT, OSTREE_FETCHER_USERAGENT_STRING);
+  curl_easy_setopt (req->easy, CURLOPT_USERAGENT,
+                    self->custom_user_agent ?: OSTREE_FETCHER_USERAGENT_STRING);
   if (self->extra_headers)
     curl_easy_setopt (req->easy, CURLOPT_HTTPHEADER, self->extra_headers);
 

src/libostree/ostree-fetcher-soup.c

@@ -374,6 +374,24 @@ session_thread_set_tls_database_cb (ThreadClosure *thread_closure,
     }
 }
 
+static void
+session_thread_set_extra_user_agent_cb (ThreadClosure *thread_closure,
+                                        gpointer data)
+{
+  const char *extra_user_agent = data;
+  if (extra_user_agent != NULL)
+    {
+      g_autofree char *ua =
+        g_strdup_printf ("%s %s", OSTREE_FETCHER_USERAGENT_STRING, extra_user_agent);
+      g_object_set (thread_closure->session, SOUP_SESSION_USER_AGENT, ua, NULL);
+    }
+  else
+    {
+      g_object_set (thread_closure->session, SOUP_SESSION_USER_AGENT,
+                    OSTREE_FETCHER_USERAGENT_STRING, NULL);
+    }
+}
+
 static void
 on_request_sent (GObject        *object, GAsyncResult   *result, gpointer        user_data);
 
@@ -774,6 +792,16 @@ _ostree_fetcher_set_extra_headers (OstreeFetcher *self,
                            (GDestroyNotify) g_variant_unref);
 }
 
+void
+_ostree_fetcher_set_extra_user_agent (OstreeFetcher *self,
+                                      const char    *extra_user_agent)
+{
+  session_thread_idle_add (self->thread_closure,
+                           session_thread_set_extra_user_agent_cb,
+                           g_strdup (extra_user_agent),
+                           (GDestroyNotify) g_free);
+}
+
 static gboolean
 finish_stream (OstreeFetcherPendingURI *pending,
                GCancellable            *cancellable,

src/libostree/ostree-fetcher.h

@@ -114,6 +114,9 @@ void _ostree_fetcher_set_tls_database (OstreeFetcher *self,
 void _ostree_fetcher_set_extra_headers (OstreeFetcher *self,
                                         GVariant      *extra_headers);
 
+void _ostree_fetcher_set_extra_user_agent (OstreeFetcher *self,
+                                           const char    *extra_user_agent);
+
 guint64 _ostree_fetcher_bytes_transferred (OstreeFetcher       *self);
 
 void _ostree_fetcher_request_to_tmpfile (OstreeFetcher         *self,

src/libostree/ostree-repo-pull.c

@@ -87,6 +87,7 @@ typedef struct {
   OstreeAsyncProgress *progress;
 
   GVariant         *extra_headers;
+  char             *append_user_agent;
 
   gboolean      dry_run;
   gboolean      dry_run_emitted_progress;
@@ -2922,11 +2923,13 @@ repo_remote_fetch_summary (OstreeRepo    *self,
   const char *url_override = NULL;
   g_autoptr(GVariant) extra_headers = NULL;
   g_autoptr(GPtrArray) mirrorlist = NULL;
+  const char *append_user_agent = NULL;
 
   if (options)
     {
       (void) g_variant_lookup (options, "override-url", "&s", &url_override);
       (void) g_variant_lookup (options, "http-headers", "@a(ss)", &extra_headers);
+      (void) g_variant_lookup (options, "append-user-agent", "&s", &append_user_agent);
     }
 
   mainctx = g_main_context_new ();
@@ -2939,6 +2942,9 @@ repo_remote_fetch_summary (OstreeRepo    *self,
   if (extra_headers)
     _ostree_fetcher_set_extra_headers (fetcher, extra_headers);
 
+  if (append_user_agent)
+    _ostree_fetcher_set_extra_user_agent (fetcher, append_user_agent);
+
   {
     g_autofree char *url_string = NULL;
     if (metalink_url_string)
@@ -3055,6 +3061,9 @@ reinitialize_fetcher (OtPullData *pull_data, const char *remote_name,
   if (pull_data->extra_headers)
     _ostree_fetcher_set_extra_headers (pull_data->fetcher, pull_data->extra_headers);
 
+  if (pull_data->append_user_agent)
+    _ostree_fetcher_set_extra_user_agent (pull_data->fetcher, pull_data->append_user_agent);
+
   return TRUE;
 }
 
@@ -3240,6 +3249,7 @@ initiate_request (OtPullData                 *pull_data,
  *   * http-headers (a(ss)): Additional headers to add to all HTTP requests
  *   * update-frequency (u): Frequency to call the async progress callback in milliseconds, if any; only values higher than 0 are valid
  *   * localcache-repos (as): File paths for local repos to use as caches when doing remote fetches
+ *   * append-user-agent (s): Additional string to append to the user agent
  */
 gboolean
 ostree_repo_pull_with_options (OstreeRepo             *self,
@@ -3311,6 +3321,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
       (void) g_variant_lookup (options, "update-frequency", "u", &update_frequency);
       (void) g_variant_lookup (options, "localcache-repos", "^a&s", &opt_localcache_repos);
       (void) g_variant_lookup (options, "timestamp-check", "b", &pull_data->timestamp_check);
+      (void) g_variant_lookup (options, "append-user-agent", "s", &pull_data->append_user_agent);
 
       if (pull_data->remote_refspec_name != NULL)
         pull_data->remote_name = g_strdup (pull_data->remote_refspec_name);
@@ -4323,6 +4334,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   g_clear_pointer (&pull_data->localcache_repos, (GDestroyNotify)g_ptr_array_unref);
   g_clear_object (&pull_data->remote_repo_local);
   g_free (pull_data->remote_name);
+  g_free (pull_data->append_user_agent);
   g_clear_pointer (&pull_data->meta_mirrorlist, (GDestroyNotify) g_ptr_array_unref);
   g_clear_pointer (&pull_data->content_mirrorlist, (GDestroyNotify) g_ptr_array_unref);
   g_clear_pointer (&pull_data->summary_data, (GDestroyNotify) g_bytes_unref);
@@ -5502,6 +5514,7 @@ ostree_repo_pull_from_remotes_async (OstreeRepo                           *self,
       copy_option (&options_dict, &local_options_dict, "http-headers", G_VARIANT_TYPE ("a(ss)"));
       copy_option (&options_dict, &local_options_dict, "subdirs", G_VARIANT_TYPE ("as"));
       copy_option (&options_dict, &local_options_dict, "update-frequency", G_VARIANT_TYPE ("u"));
+      copy_option (&options_dict, &local_options_dict, "append-user-agent", G_VARIANT_TYPE ("s"));
 
       local_options = g_variant_dict_end (&local_options_dict);
 
@@ -5725,6 +5738,7 @@ ostree_repo_resolve_keyring_for_collection (OstreeRepo    *self,
  *
  * - override-url (s): Fetch summary from this URL if remote specifies no metalink in options
  * - http-headers (a(ss)): Additional headers to add to all HTTP requests
+ * - append-user-agent (s): Additional string to append to the user agent
  *
  * Returns: %TRUE on success, %FALSE on failure
  */

src/ostree/ot-builtin-pull.c

@@ -41,6 +41,7 @@ static gboolean opt_bareuseronly_files;
 static char** opt_subpaths;
 static char** opt_http_headers;
 static char* opt_cache_dir;
+static char* opt_append_user_agent;
 static int opt_depth = 0;
 static int opt_frequency = 0;
 static char* opt_url;
@@ -69,6 +70,8 @@ static GOptionEntry options[] = {
    { "update-frequency", 0, 0, G_OPTION_ARG_INT, &opt_frequency, "Sets the update frequency, in milliseconds (0=1000ms) (default: 0)", "FREQUENCY" },
    { "localcache-repo", 'L', 0, G_OPTION_ARG_FILENAME_ARRAY, &opt_localcache_repos, "Add REPO as local cache source for objects during this pull", "REPO" },
    { "timestamp-check", 'T', 0, G_OPTION_ARG_NONE, &opt_timestamp_check, "Require fetched commits to have newer timestamps", NULL },
+   /* let's leave this hidden for now; we just need it for tests */
+   { "append-user-agent", 0, G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_STRING, &opt_append_user_agent, "Append string to user agent", NULL },
    { NULL }
  };
 
@@ -333,6 +336,10 @@ ostree_builtin_pull (int argc, char **argv, OstreeCommandInvocation *invocation,
                                g_variant_new_variant (g_variant_builder_end (&hdr_builder)));
       }
 
+    if (opt_append_user_agent)
+      g_variant_builder_add (&builder, "{s@v}", "append-user-agent",
+                             g_variant_new_variant (g_variant_new_string (opt_append_user_agent)));
+
     if (!opt_dry_run)
       {
         if (console.is_tty)

tests/libtest.sh

@@ -234,10 +234,9 @@ ostree_repo_init() {
 # The original one; use setup_fake_remote_repo2 for newer code,
 # down the line we'll try to port tests.
 setup_fake_remote_repo1() {
-    mode=$1
-    commit_opts=${2:-}
-    args=${3:-}
-    shift
+    mode=$1; shift
+    commit_opts=${1:-}
+    [ $# -eq 0 ] || shift
     oldpwd=`pwd`
     mkdir ostree-srv
     cd ostree-srv
@@ -263,7 +262,7 @@ setup_fake_remote_repo1() {
     mkdir ${test_tmpdir}/httpd
     cd httpd
     ln -s ${test_tmpdir}/ostree-srv ostree
-    ${OSTREE_HTTPD} --autoexit --log-file $(pwd)/httpd.log --daemonize -p ${test_tmpdir}/httpd-port $args
+    ${OSTREE_HTTPD} --autoexit --log-file $(pwd)/httpd.log --daemonize -p ${test_tmpdir}/httpd-port "$@"
     port=$(cat ${test_tmpdir}/httpd-port)
     echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address
     cd ${oldpwd} 

tests/test-remote-cookies.sh

@@ -27,7 +27,8 @@ echo '1..4'
 . $(dirname $0)/libtest.sh
 
 setup_fake_remote_repo1 "archive" "" \
-  "--expected-cookies foo=bar --expected-cookies baz=badger"
+  --expected-cookies foo=bar \
+  --expected-cookies baz=badger
 
 assert_fail (){ 
   if $@; then

tests/test-remote-headers.sh

@@ -25,8 +25,13 @@ echo '1..2'
 
 . $(dirname $0)/libtest.sh
 
+V=$($CMD_PREFIX ostree --version | \
+  python3 -c 'import sys, yaml; print(yaml.safe_load(sys.stdin)["libostree"]["Version"])')
+
 setup_fake_remote_repo1 "archive" "" \
-  "--expected-header foo=bar --expected-header baz=badger"
+  --expected-header foo=bar \
+  --expected-header baz=badger \
+  --expected-header "User-Agent=libostree/$V dodo/2.15"
 
 assert_fail (){
   set +e
@@ -46,9 +51,22 @@ ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat
 # Sanity check the setup, without headers the pull should fail
 assert_fail ${CMD_PREFIX} ostree --repo=repo pull origin main
 
+# without proper User-Agent, the pull should fail
+assert_fail ${CMD_PREFIX} ostree --repo=repo pull origin main \
+  --http-header foo=bar \
+  --http-header baz=badger
+assert_fail ${CMD_PREFIX} ostree --repo=repo pull origin main \
+  --http-header foo=bar \
+  --http-header baz=badger \
+  --append-user-agent bar/1.2
+
 echo "ok setup done"
 
 # Now pull should succeed now
-${CMD_PREFIX} ostree --repo=repo pull --http-header foo=bar --http-header baz=badger origin main
+${CMD_PREFIX} ostree --repo=repo pull \
+  --http-header foo=bar \
+  --http-header baz=badger \
+  --append-user-agent dodo/2.15 \
+  origin main
 
 echo "ok pull succeeded"