[gnome-software] Add external-appstream plugin

[Richard Hughes <rhughes src gnome org>]

commit 45926af0f74e760c418deebb163f6e8367b4e430
Author: Joaquim Rocha <jrocha endlessm com>
Date:   Wed Oct 12 19:02:48 2016 +0100

    Add external-appstream plugin
    
    This plugin is used to retrieve appstream files from external locations.
    It can be used by vendors that do not have control over the applications
    they ship but still want to provide a more personalized experience
    through extra appstream files that are updated independently from their
    distro.
    
    The external appstream files get downloaded and then moved to a system location
    using a pkexec helper.
    
    Signed-off-by: Richard Hughes <richard hughsie com>

 configure.ac                                       |    8 +
 contrib/gnome-software.spec.in                     |    3 +
 data/Makefile.am                                   |   13 +-
 ...org.gnome.software.external-appstream.policy.in |   20 ++
 data/org.gnome.software.gschema.xml                |    4 +
 src/plugins/Makefile.am                            |   23 ++
 src/plugins/gnome-software-install-appstream.in    |    8 +
 src/plugins/gs-plugin-external-appstream.c         |  230 ++++++++++++++++++++
 8 files changed, 308 insertions(+), 1 deletions(-)
---
diff --git a/configure.ac b/configure.ac
index f115c23..321b81f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -418,6 +418,13 @@ AS_IF([test "x$have_snap" = "xyes"], [
 ])
 AM_CONDITIONAL(HAVE_SNAP, test "$have_snap" != no)
 
+# External appstream
+AC_ARG_ENABLE(external-appstream,
+              [AS_HELP_STRING([--enable-external-appstream],
+                              [enable External Appstream support [default=yes]])],,
+              enable_external_appstream=yes)
+AM_CONDITIONAL(HAVE_EXTERNAL_APPSTREAM, test "x$enable_external_appstream" = "xyes")
+
 GLIB_TESTS
 
 dnl ---------------------------------------------------------------------------
@@ -475,4 +482,5 @@ echo "
         Webapps support:           ${enable_webapps}
         Ubuntu Reviews support:    ${enable_ubuntu_reviews}
         Snap support:              ${have_snap}
+        Ext. appstream support:    ${enable_external_appstream}
 "
diff --git a/contrib/gnome-software.spec.in b/contrib/gnome-software.spec.in
index 54a46bd..f60586b 100644
--- a/contrib/gnome-software.spec.in
+++ b/contrib/gnome-software.spec.in
@@ -155,6 +155,7 @@ glib-compile-schemas %{_datadir}/glib-2.0/schemas &> /dev/null || :
 %{_libdir}/gs-plugins-%{gs_plugin_version}/libgs_plugin_dpkg.so
 %{_libdir}/gs-plugins-%{gs_plugin_version}/libgs_plugin_dummy.so
 %{_libdir}/gs-plugins-%{gs_plugin_version}/libgs_plugin_epiphany.so
+%{_libdir}/gs-plugins-%{gs_plugin_version}/libgs_plugin_external-appstream.so
 %{_libdir}/gs-plugins-%{gs_plugin_version}/libgs_plugin_fedora-distro-upgrades.so
 %{_libdir}/gs-plugins-%{gs_plugin_version}/libgs_plugin_fedora-tagger-usage.so
 %{_libdir}/gs-plugins-%{gs_plugin_version}/libgs_plugin_flatpak-system.so
@@ -197,6 +198,8 @@ glib-compile-schemas %{_datadir}/glib-2.0/schemas &> /dev/null || :
 %{_datadir}/gnome-shell/search-providers/org.gnome.Software-search-provider.ini
 %{_datadir}/glib-2.0/schemas/org.gnome.software.gschema.xml
 %{_datadir}/glib-2.0/schemas/org.gnome.software-fedora.gschema.override
+%{_datadir}/polkit-1/actions/org.gnome.software.external-appstream.policy
+%{_libexecdir}/gnome-software-install-appstream
 
 %files devel
 %{_libdir}/pkgconfig/gnome-software.pc
diff --git a/data/Makefile.am b/data/Makefile.am
index 4ac08a7..7f2103c 100644
--- a/data/Makefile.am
+++ b/data/Makefile.am
@@ -39,9 +39,20 @@ install-sample-data:
        rm fedora-20-icons.tar.gz; \
        cd -
 
-EXTRA_DIST =                                           \
+polkit_policydir = $(datadir)/polkit-1/actions
+polkit_policy_in_files = org.gnome.software.external-appstream.policy.in
+polkit_policy_DATA = $(polkit_policy_in_files:.policy.in=.policy)
+
+%.policy: %.policy.in Makefile
+       $(AM_V_GEN) sed -e "s|@libexecdir[@]|$(libexecdir)|g" $< > $@
+
+EXTRA_DIST =                                                   \
+       org.gnome.software.external-appstream.policy.in         \
        $(gsettings_SCHEMAS)
 
+CLEANFILES =                                           \
+       org.gnome.software.external-appstream.policy
+
 MAINTAINERCLEANFILES =                                 \
        *~                                              \
        Makefile.in
diff --git a/data/org.gnome.software.external-appstream.policy.in 
b/data/org.gnome.software.external-appstream.policy.in
new file mode 100644
index 0000000..69af72a
--- /dev/null
+++ b/data/org.gnome.software.external-appstream.policy.in
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!DOCTYPE policyconfig PUBLIC
+ "-//freedesktop//DTD PolicyKit Policy Configuration 1.0//EN"
+ "http://www.freedesktop.org/standards/PolicyKit/1.0/policyconfig.dtd";>
+<policyconfig>
+  <vendor>GNOME</vendor>
+  <vendor_url>https://www.gnome.org/</vendor_url>
+
+  <action id="org.gnome.software.install-appstream">
+    <description>Install an appstream file into a system location</description>
+    <message>Installing an appstream file into a system location</message>
+    <defaults>
+      <allow_any>yes</allow_any>
+      <allow_active>yes</allow_active>
+      <allow_inactive>no</allow_inactive>
+    </defaults>
+    <annotate 
key="org.freedesktop.policykit.exec.path">@libexecdir@/gnome-software-install-appstream</annotate>
+  </action>
+</policyconfig>
diff --git a/data/org.gnome.software.gschema.xml b/data/org.gnome.software.gschema.xml
index c33924f..b291f45 100644
--- a/data/org.gnome.software.gschema.xml
+++ b/data/org.gnome.software.gschema.xml
@@ -110,5 +110,9 @@
       <default>[]</default>
       <summary>A list of non-free sources that can be optionally enabled</summary>
     </key>
+    <key name="external-appstream-urls" type="as">
+      <default>[]</default>
+      <summary>A list of URLs pointing to appstream files that will be downloaded into an app-info 
folder</summary>
+    </key>
   </schema>
 </schemalist>
diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am
index ad5e63e..eed5048 100644
--- a/src/plugins/Makefile.am
+++ b/src/plugins/Makefile.am
@@ -23,6 +23,7 @@ AM_CPPFLAGS =                                         \
        -DDATADIR=\"$(datadir)\"                        \
        -DG_LOG_DOMAIN=\"GsPlugin\"                     \
        -DLIBDIR=\""$(libdir)"\"                        \
+       -DLIBEXECDIR=\""$(libexecdir)"\"                \
        -DLOCALSTATEDIR=\""$(localstatedir)"\"          \
        -DSBINDIR=\"$(sbindir)\"                        \
        -DSYSCONFDIR=\""$(sysconfdir)"\"                \
@@ -48,6 +49,13 @@ plugin_LTLIBRARIES =                                 \
        libgs_plugin_icons.la                           \
        libgs_plugin_ubuntuone.la
 
+if HAVE_EXTERNAL_APPSTREAM
+plugin_LTLIBRARIES += libgs_plugin_external-appstream.la
+
+libexec_SCRIPTS =                      \
+       gnome-software-install-appstream
+endif
+
 if HAVE_SNAP
 plugin_LTLIBRARIES += libgs_plugin_snap.la
 endif
@@ -121,6 +129,21 @@ libgs_plugin_dpkg_la_LIBADD = $(GS_PLUGIN_LIBS)
 libgs_plugin_dpkg_la_LDFLAGS = -module -avoid-version
 libgs_plugin_dpkg_la_CFLAGS = $(GS_PLUGIN_CFLAGS) $(WARN_CFLAGS)
 
+libgs_plugin_external_appstream_la_SOURCES =           \
+       gs-plugin-external-appstream.c
+libgs_plugin_external_appstream_la_LIBADD = $(GS_PLUGIN_LIBS) $(SOUP_LIBS)
+libgs_plugin_external_appstream_la_LDFLAGS = -module -avoid-version
+libgs_plugin_external_appstream_la_CFLAGS = $(GS_PLUGIN_CFLAGS) $(WARN_CFLAGS) $(SOUP_CFLAGS)
+
+gnome-software-install-appstream: gnome-software-install-appstream.in Makefile
+       $(AM_V_GEN) sed -e "s|@localstatedir[@]|$(localstatedir)|g" $< > $@
+
+EXTRA_DIST =                                           \
+       gnome-software-install-appstream.in
+
+CLEANFILES =                                           \
+       gnome-software-install-appstream
+
 libgs_plugin_key_colors_la_SOURCES = gs-plugin-key-colors.c
 libgs_plugin_key_colors_la_LIBADD = $(GS_PLUGIN_LIBS)
 libgs_plugin_key_colors_la_LDFLAGS = -module -avoid-version
diff --git a/src/plugins/gnome-software-install-appstream.in b/src/plugins/gnome-software-install-appstream.in
new file mode 100644
index 0000000..b0fa04e
--- /dev/null
+++ b/src/plugins/gnome-software-install-appstream.in
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+
+APPSTREAM_FILE=$1
+TARGET_FILE=$2
+APPSTREAM_SYSTEM_LOCATION=@localstatedir@/cache/app-info/xmls
+
+mkdir -p "${APPSTREAM_SYSTEM_LOCATION}"
+mv "${APPSTREAM_FILE}" "${APPSTREAM_SYSTEM_LOCATION}/${TARGET_FILE}"
diff --git a/src/plugins/gs-plugin-external-appstream.c b/src/plugins/gs-plugin-external-appstream.c
new file mode 100644
index 0000000..474ee2d
--- /dev/null
+++ b/src/plugins/gs-plugin-external-appstream.c
@@ -0,0 +1,230 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ *
+ * Copyright (C) 2016 Endless Mobile, Inc.
+ *
+ * Authors: Joaquim Rocha <jrocha endlessm com>
+ *
+ * Licensed under the GNU General Public License Version 2
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <config.h>
+
+#include <gnome-software.h>
+
+#define APPSTREAM_SYSTEM_DIR LOCALSTATEDIR "/cache/app-info/xmls"
+
+struct GsPluginData {
+       GSettings *settings;
+};
+
+void
+gs_plugin_initialize (GsPlugin *plugin)
+{
+       GsPluginData *priv = gs_plugin_alloc_data (plugin, sizeof(GsPluginData));
+
+       priv->settings = g_settings_new ("org.gnome.software");
+
+       /* run it before the appstream plugin */
+       gs_plugin_add_rule (plugin, GS_PLUGIN_RULE_RUN_BEFORE, "appstream");
+
+       g_debug ("appstream system dir: %s", APPSTREAM_SYSTEM_DIR);
+}
+
+void
+gs_plugin_destroy (GsPlugin *plugin)
+{
+       GsPluginData *priv = gs_plugin_get_data (plugin);
+       g_object_unref (priv->settings);
+}
+
+static gboolean
+gs_plugin_external_appstream_check (const gchar *appstream_path,
+                                           guint cache_age)
+{
+       g_autoptr(GFile) file = g_file_new_for_path (appstream_path);
+       guint appstream_file_age = gs_utils_get_file_age (file);
+       return appstream_file_age >= cache_age;
+}
+
+static gboolean
+gs_plugin_external_appstream_install (const gchar *appstream_file,
+                                     const gchar *target_file_name,
+                                     GCancellable *cancellable,
+                                     GError **error)
+{
+       g_autoptr(GSubprocess) subprocess = NULL;
+       const gchar *argv[] = { "pkexec",
+                               LIBEXECDIR "/gnome-software-install-appstream",
+                               appstream_file, target_file_name, NULL};
+       g_debug ("Installing the appstream file %s in the system",
+                appstream_file);
+       subprocess = g_subprocess_newv (argv,
+                                       G_SUBPROCESS_FLAGS_STDOUT_PIPE |
+                                       G_SUBPROCESS_FLAGS_STDIN_PIPE, error);
+       if (subprocess == NULL)
+               return FALSE;
+       return g_subprocess_wait_check (subprocess, cancellable, error);
+}
+
+static gchar *
+gs_plugin_external_appstream_get_modification_date (const gchar *file_path)
+{
+       GTimeVal time_val;
+       g_autoptr(GDateTime) date_time = NULL;
+       g_autoptr(GFile) file = NULL;
+       g_autoptr(GFileInfo) info = NULL;
+
+       file = g_file_new_for_path (file_path);
+       info = g_file_query_info (file,
+                                 G_FILE_ATTRIBUTE_TIME_MODIFIED,
+                                 G_FILE_QUERY_INFO_NONE,
+                                 NULL,
+                                 NULL);
+       if (info == NULL)
+               return NULL;
+       g_file_info_get_modification_time (info, &time_val);
+       date_time = g_date_time_new_from_timeval_local (&time_val);
+       return g_date_time_format (date_time, "%a, %d %b %Y %H:%M:%S %Z");
+}
+
+static gboolean
+gs_plugin_external_appstream_refresh_url (GsPlugin *plugin,
+                                         const gchar *url,
+                                         guint cache_age,
+                                         GCancellable *cancellable,
+                                         GError **error)
+{
+       GOutputStream *outstream = NULL;
+       SoupSession *soup_session;
+       const gchar *tmp_file_path = NULL;
+       guint status_code;
+       g_autofree gchar *file_name = NULL;
+       g_autofree gchar *local_mod_date = NULL;
+       g_autofree gchar *target_file_path = NULL;
+       g_autofree gchar *tmp_file_tmpl = NULL;
+       g_autoptr(GFileIOStream) iostream = NULL;
+       g_autoptr(GFile) tmp_file = NULL;
+       g_autoptr(SoupMessage) msg = NULL;
+
+       /* check age */
+       file_name = g_path_get_basename (url);
+       target_file_path = g_build_filename (APPSTREAM_SYSTEM_DIR, file_name, NULL);
+       if (!gs_plugin_external_appstream_check (target_file_path, cache_age)) {
+               g_debug ("skipping updating external appstream file %s: "
+                        "cache age is older than file",
+                        target_file_path);
+               return TRUE;
+       }
+
+       msg = soup_message_new (SOUP_METHOD_GET, url);
+
+       /* Set the If-Modified-Since header if the target file exists */
+       target_file_path = g_build_filename (APPSTREAM_SYSTEM_DIR, file_name,
+                                            NULL);
+       local_mod_date = gs_plugin_external_appstream_get_modification_date (target_file_path);
+       if (local_mod_date != NULL) {
+               g_debug ("Requesting contents of %s if modified since %s",
+                        url, local_mod_date);
+               soup_message_headers_append (msg->request_headers,
+                                            "If-Modified-Since",
+                                            local_mod_date);
+       }
+
+       /* get the data */
+       soup_session = gs_plugin_get_soup_session (plugin);
+       status_code = soup_session_send_message (soup_session, msg);
+       if (status_code != SOUP_STATUS_OK) {
+               if (status_code == SOUP_STATUS_NOT_MODIFIED) {
+                       g_debug ("Not updating %s has not modified since %s",
+                                target_file_path, local_mod_date);
+                       return TRUE;
+               }
+
+               g_set_error (error, GS_PLUGIN_ERROR,
+                            GS_PLUGIN_ERROR_DOWNLOAD_FAILED,
+                            "Failed to download appstream file %s: %s",
+                            url, soup_status_get_phrase (status_code));
+               return FALSE;
+       }
+
+       /* write the download contents into a temporary file that will be
+        * moved into the system */
+       tmp_file_tmpl = g_strdup_printf ("XXXXXX_%s", file_name);
+       tmp_file = g_file_new_tmp (tmp_file_tmpl, &iostream, error);
+       if (tmp_file == NULL)
+               return FALSE;
+
+       tmp_file_path = g_file_get_path (tmp_file);
+       g_debug ("Downloaded appstream file %s", tmp_file_path);
+
+       /* write to file */
+       outstream = g_io_stream_get_output_stream (G_IO_STREAM (iostream));
+       if (!g_output_stream_write_all (outstream, msg->response_body->data,
+                                       msg->response_body->length, NULL,
+                                       cancellable, error)) {
+               /* clean up the temporary file if we could not write */
+               g_file_delete (tmp_file, NULL, NULL);
+               return FALSE;
+       }
+
+       /* install file systemwide */
+       if (!gs_plugin_external_appstream_install (tmp_file_path,
+                                                  file_name,
+                                                  cancellable,
+                                                  error))
+               return FALSE;
+       g_debug ("Installed appstream file %s as %s", tmp_file_path, file_name);
+
+       return TRUE;
+}
+
+gboolean
+gs_plugin_refresh (GsPlugin *plugin,
+                  guint cache_age,
+                  GsPluginRefreshFlags flags,
+                  GCancellable *cancellable,
+                  GError **error)
+{
+       GsPluginData *priv = gs_plugin_get_data (plugin);
+       guint i;
+       g_auto(GStrv) appstream_urls = NULL;
+
+       if ((flags & GS_PLUGIN_REFRESH_FLAGS_METADATA) == 0)
+               return TRUE;
+
+       appstream_urls = g_settings_get_strv (priv->settings,
+                                             "external-appstream-urls");
+       for (i = 0; appstream_urls[i] != NULL; ++i) {
+               g_autoptr(GError) local_error = NULL;
+               if (!g_str_has_prefix (appstream_urls[i], "https")) {
+                       g_warning ("Not considering %s as an external "
+                                  "appstream source: please use an https URL",
+                                  appstream_urls[i]);
+                       continue;
+               }
+               if (!gs_plugin_external_appstream_refresh_url (plugin,
+                                                              appstream_urls[i],
+                                                              cache_age,
+                                                              cancellable,
+                                                              &local_error)) {
+                       g_warning ("Failed to update external appstream file: %s",
+                                  local_error->message);
+               }
+       }
+
+       return TRUE;
+}