lib/core: add ostree_checksum_file_at API

This is like `ostree_checksum_file` but fd-relative. This will be used
by https://github.com/ostreedev/ostree/pull/1258.

AFAICT, we actually didn't have any tests that check the `checksum` CLI.
Add a basic one here to test the old code as well as the new code.

Closes: #1263
Approved by: cgwalters

src/libostree/libostree-devel.sym

@@ -19,6 +19,8 @@
 
 /* Add new symbols here.  Release commits should copy this section into -released.sym. */
 LIBOSTREE_2017.13 {
+global:
+  ostree_checksum_file_at;
 } LIBOSTREE_2017.12;
 
 /* Stub section for the stable release *after* this development one; don't
@@ -27,6 +29,6 @@ LIBOSTREE_2017.13 {
  * with whatever the next version with new symbols will be.
 LIBOSTREE_2017.$NEWVERSION {
 global:
-	someostree_symbol_deleteme;
+  someostree_symbol_deleteme;
 } LIBOSTREE_2017.$LASTSTABLE;
 */

src/libostree/ostree-core.c

@@ -832,6 +832,81 @@ ostree_checksum_file (GFile            *f,
   return TRUE;
 }
 
+/**
+ * ostree_checksum_file_at:
+ * @dfd: Directory file descriptor
+ * @path: Subpath
+ * @stbuf (allow-none): Optional stat buffer
+ * @objtype: Object type
+ * @flags: Flags
+ * @out_checksum (out) (transfer full): Return location for hex checksum
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * Compute the OSTree checksum for a given file. This is an fd-relative version
+ * of ostree_checksum_file() which also takes flags and fills in a caller
+ * allocated buffer.
+ *
+ * Since: 2017.13
+ */
+gboolean
+ostree_checksum_file_at (int               dfd,
+                         const char       *path,
+                         struct stat      *stbuf,
+                         OstreeObjectType  objtype,
+                         OstreeChecksumFlags flags,
+                         char            **out_checksum,
+                         GCancellable     *cancellable,
+                         GError          **error)
+{
+  g_return_val_if_fail (out_checksum != NULL, FALSE);
+
+  if (g_cancellable_set_error_if_cancelled (cancellable, error))
+    return FALSE;
+
+  struct stat local_stbuf;
+  if (stbuf == NULL)
+    {
+      stbuf = &local_stbuf;
+      if (!glnx_fstatat (dfd, path, stbuf, AT_SYMLINK_NOFOLLOW, error))
+        return FALSE;
+    }
+
+  g_autoptr(GFileInfo) file_info = _ostree_stbuf_to_gfileinfo (stbuf);
+
+  g_autoptr(GInputStream) in = NULL;
+  if (S_ISREG (stbuf->st_mode))
+    {
+      glnx_autofd int fd = -1;
+      if (!glnx_openat_rdonly (dfd, path, FALSE, &fd, error))
+        return FALSE;
+      in = g_unix_input_stream_new (glnx_steal_fd (&fd), TRUE);
+    }
+  else if (S_ISLNK (stbuf->st_mode))
+    {
+      if (!ot_readlinkat_gfile_info (dfd, path, file_info, cancellable, error))
+        return FALSE;
+    }
+
+  const gboolean ignore_xattrs =
+    ((flags & OSTREE_CHECKSUM_FLAGS_IGNORE_XATTRS) > 0);
+
+  g_autoptr(GVariant) xattrs = NULL;
+  if (!ignore_xattrs && objtype == OSTREE_OBJECT_TYPE_FILE)
+    {
+      if (!glnx_dfd_name_get_all_xattrs (dfd, path, &xattrs, cancellable, error))
+        return FALSE;
+    }
+
+  g_autofree guchar *csum_bytes = NULL;
+  if (!ostree_checksum_file_from_input (file_info, xattrs, in, objtype,
+                                        &csum_bytes, cancellable, error))
+    return FALSE;
+
+  *out_checksum = ostree_checksum_from_bytes (csum_bytes);
+  return TRUE;
+}
+
 typedef struct {
   GFile  *f;
   OstreeObjectType objtype;

src/libostree/ostree-core.h

@@ -21,6 +21,7 @@
 
 #pragma once
 
+#include <sys/stat.h>
 #include <gio/gio.h>
 #include <ostree-types.h>
 
@@ -420,6 +421,26 @@ gboolean ostree_checksum_file (GFile             *f,
                                GCancellable      *cancellable,
                                GError           **error);
 
+/**
+ * OstreeChecksumFlags:
+ *
+ * Since: 2017.13
+ */
+typedef enum {
+  OSTREE_CHECKSUM_FLAGS_NONE = 0,
+  OSTREE_CHECKSUM_FLAGS_IGNORE_XATTRS = (1 << 0),
+} OstreeChecksumFlags;
+
+_OSTREE_PUBLIC
+gboolean ostree_checksum_file_at (int               dfd,
+                                  const char       *path,
+                                  struct stat      *stbuf,
+                                  OstreeObjectType  objtype,
+                                  OstreeChecksumFlags flags,
+                                  char            **out_checksum,
+                                  GCancellable     *cancellable,
+                                  GError          **error);
+
 _OSTREE_PUBLIC
 void ostree_checksum_file_async (GFile                 *f,
                                  OstreeObjectType       objtype,

src/ostree/ot-builtin-checksum.c

@@ -32,7 +32,10 @@
  * man page (man/ostree-checksum.xml) when changing the option list.
  */
 
+static gboolean opt_ignore_xattrs;
+
 static GOptionEntry options[] = {
+  { "ignore-xattrs", 0, 0, G_OPTION_ARG_NONE, &opt_ignore_xattrs, "Don't include xattrs in checksum", NULL },
   { NULL }
 };
 
@@ -49,12 +52,14 @@ on_checksum_received (GObject    *obj,
 {
   AsyncChecksumData *data = user_data;
 
-  g_autofree guchar *csum = NULL;
-  data->success = ostree_checksum_file_async_finish ((GFile*)obj, result, &csum, data->error);
+  g_autofree guchar *csum_bytes = NULL;
+  data->success =
+    ostree_checksum_file_async_finish ((GFile*)obj, result, &csum_bytes, data->error);
   if (data->success)
     {
-      g_autofree char *checksum = ostree_checksum_from_bytes (csum);
-      g_print ("%s\n", checksum);
+      char csum[OSTREE_SHA256_STRING_LEN+1];
+      ostree_checksum_inplace_from_bytes (csum_bytes, csum);
+      g_print ("%s\n", csum);
     }
 
   g_main_loop_quit (data->loop);
@@ -73,15 +78,28 @@ ostree_builtin_checksum (int argc, char **argv, GCancellable *cancellable, GErro
     return glnx_throw (error, "A filename must be given");
   const char *path = argv[1];
 
-  g_autoptr(GFile) f = g_file_new_for_path (path);
-  g_autoptr(GMainLoop) loop = g_main_loop_new (NULL, FALSE);
+  /* for test coverage, use the async API if no flags are needed */
+  if (!opt_ignore_xattrs)
+    {
+      g_autoptr(GFile) f = g_file_new_for_path (path);
+      g_autoptr(GMainLoop) loop = g_main_loop_new (NULL, FALSE);
 
-  AsyncChecksumData data = { 0, };
+      AsyncChecksumData data = { 0, };
 
-  data.loop = loop;
-  data.error = error;
-  ostree_checksum_file_async (f, OSTREE_OBJECT_TYPE_FILE, G_PRIORITY_DEFAULT, cancellable,
-                              on_checksum_received, &data);
-  g_main_loop_run (data.loop);
-  return data.success;
+      data.loop = loop;
+      data.error = error;
+      ostree_checksum_file_async (f, OSTREE_OBJECT_TYPE_FILE, G_PRIORITY_DEFAULT,
+                                  cancellable, on_checksum_received, &data);
+      g_main_loop_run (data.loop);
+      return data.success;
+    }
+
+  g_autofree char *checksum = NULL;
+  if (!ostree_checksum_file_at (AT_FDCWD, path, NULL, OSTREE_OBJECT_TYPE_FILE,
+                                OSTREE_CHECKSUM_FLAGS_IGNORE_XATTRS, &checksum,
+                                cancellable, error))
+    return FALSE;
+
+  g_print ("%s\n", checksum);
+  return TRUE;
 }