Use XMLb to query quirks

During startup we do 1898 persistent allocations to load the quirk files, which
equates to ~90kb of RSS. Use libxmlb to create a mmap'able store we can query
with XPath queries at runtime.

.lgtm.yml

@@ -12,7 +12,7 @@ extraction:
       - python3-cairo
       - libssl-dev
     after_prepare:
-    - "wget -O libxmlb.zip https://github.com/hughsie/libxmlb/archive/0.1.7.zip"
+    - "wget -O libxmlb.zip https://github.com/hughsie/libxmlb/archive/0.1.13.zip"
     - "mkdir -p subprojects/libxmlb"
     - "bsdtar --strip-components=1 -xvf libxmlb.zip -C subprojects/libxmlb"
     - "wget -O flashrom.zip https://github.com/hughsie/flashrom/archive/wip/hughsie/fwupd.zip"

contrib/ci/dependencies.xml

@@ -602,20 +602,18 @@
     </distro>
     <distro id="debian">
       <control>
-        <version>(>= 0.1.5)</version>
+        <version>(>= 0.1.13)</version>
       </control>
       <package variant="x86_64" />
       <package variant="s390x">libxmlb-dev:s390x</package>
       <package variant="i386" />
     </distro>
-<!--
     <distro id="ubuntu">
       <control>
-        <version>(>= 0.1.5)</version>
+        <version>(>= 0.1.13)</version>
       </control>
       <package variant="x86_64" />
     </distro>
--->
   </dependency>
   <dependency type="build" id="libarchive-dev">
     <distro id="centos">

data/tests/quirks.d/merged.quirk

@@ -1,2 +0,0 @@
-[USB\VID_0A5C&PID_6412]
-Flags = MERGE_ME

data/tests/quirks.d/tests.quirk

@@ -1,9 +1,6 @@
 [USB\VID_0A5C&PID_6412]
 Flags= ignore-runtime
 
-[USB\VID_FFFF&PID_FFFF]
-Flags =
-
 [ACME Inc.=True]
 Test = awesome
 

meson.build

@@ -169,7 +169,7 @@ gudev = dependency('gudev-1.0')
 if gudev.version().version_compare('>= 232')
   conf.set('HAVE_GUDEV_232', '1')
 endif
-libxmlb = dependency('xmlb', version : '>= 0.1.7', fallback : ['libxmlb', 'libxmlb_dep'])
+libxmlb = dependency('xmlb', version : '>= 0.1.13', fallback : ['libxmlb', 'libxmlb_dep'])
 gusb = dependency('gusb', version : '>= 0.2.9')
 sqlite = dependency('sqlite3')
 libarchive = dependency('libarchive')

plugins/dfu/dfu-tool.c

@@ -1287,7 +1287,7 @@ main (int argc, char *argv[])
 
 	/* use quirks */
 	priv->quirks = fu_quirks_new ();
-	if (!fu_quirks_load (priv->quirks, &error)) {
+	if (!fu_quirks_load (priv->quirks, FU_QUIRKS_LOAD_FLAG_NONE, &error)) {
 		/* TRANSLATORS: quirks are device-specific workarounds */
 		g_print ("%s: %s\n", _("Failed to load quirks"), error->message);
 		return EXIT_FAILURE;

plugins/uefi/fu-plugin-uefi.c

@@ -751,7 +751,6 @@ fu_plugin_uefi_create_dummy (FuPlugin *plugin, const gchar *reason, GError **err
 	fu_device_add_flag (dev, FWUPD_DEVICE_FLAG_REQUIRE_AC);
 
 	fu_device_add_icon (dev, "computer");
-	fu_device_set_plugin (dev, fu_plugin_get_name (plugin));
 	fu_device_set_id (dev, "UEFI-dummy");
 	fu_device_add_instance_id (dev, "main-system-firmware");
 	if (!fu_device_setup (dev, error))

src/fu-config.c

@@ -550,11 +550,9 @@ fu_config_load (FuConfig *self, FuConfigLoadFlags flags, GError **error)
 	for (guint i = 0; locales[i] != NULL; i++)
 		xb_builder_add_locale (builder, locales[i]);
 
-#if LIBXMLB_CHECK_VERSION(0,1,7)
 	/* on a read-only filesystem don't care about the cache GUID */
 	if (flags & FU_CONFIG_LOAD_FLAG_READONLY_FS)
 		compile_flags |= XB_BUILDER_COMPILE_FLAG_IGNORE_GUID;
-#endif
 
 	/* build the metainfo silo */
 	cachedirpkg = fu_common_get_path (FU_PATH_KIND_CACHEDIR_PKG);

src/fu-device-private.h

@@ -9,6 +9,8 @@
 #include <fu-device.h>
 #include <xmlb.h>
 
+#define fu_device_set_plugin(d,v)		fwupd_device_set_plugin(FWUPD_DEVICE(d),v)
+
 /**
  * FuDeviceInstanceFlags:
  * @FU_DEVICE_INSTANCE_FLAG_NONE:		No flags set
@@ -46,3 +48,4 @@ void		 fu_device_add_instance_id_full		(FuDevice	*self,
 							 const gchar	*instance_id,
 							 FuDeviceInstanceFlags flags);
 gchar		*fu_device_get_guids_as_str		(FuDevice	*self);
+GPtrArray	*fu_device_get_possible_plugins		(FuDevice	*self);

src/fu-device.c

@@ -56,6 +56,7 @@ typedef struct {
 	guint64				 size_max;
 	gint				 open_refcount;	/* atomic */
 	GType				 specialized_gtype;
+	GPtrArray			*possible_plugins;
 } FuDevicePrivate;
 
 enum {
@@ -126,6 +127,40 @@ fu_device_set_property (GObject *object, guint prop_id,
 	}
 }
 
+/**
+ * fu_device_get_possible_plugins:
+ * @self: A #FuDevice
+ *
+ * Gets the list of possible plugin names, typically added from quirk files.
+ *
+ * Returns: (element-type utf-8) (transfer container): plugin names
+ *
+ * Since: 1.3.3
+ **/
+GPtrArray *
+fu_device_get_possible_plugins (FuDevice *self)
+{
+	FuDevicePrivate *priv = GET_PRIVATE (self);
+	return g_ptr_array_ref (priv->possible_plugins);
+}
+
+/**
+ * fu_device_add_possible_plugin:
+ * @self: A #FuDevice
+ * @plugin: A plugin name, e.g. `dfu`
+ *
+ * Adds a plugin name to the list of plugins that *might* be able to handle this
+ * device. This is tyically called from a quirk handler.
+ *
+ * Since: 1.3.3
+ **/
+static void
+fu_device_add_possible_plugin (FuDevice *self, const gchar *plugin)
+{
+	FuDevicePrivate *priv = GET_PRIVATE (self);
+	g_ptr_array_add (priv->possible_plugins, g_strdup (plugin));
+}
+
 /**
  * fu_device_poll:
  * @self: A #FuDevice
@@ -644,7 +679,7 @@ fu_device_set_quirk_kv (FuDevice *self,
 	FuDeviceClass *klass = FU_DEVICE_GET_CLASS (self);
 
 	if (g_strcmp0 (key, FU_QUIRKS_PLUGIN) == 0) {
-		fu_device_set_plugin (self, value);
+		fu_device_add_possible_plugin (self, value);
 		return TRUE;
 	}
 	if (g_strcmp0 (key, FU_QUIRKS_FLAGS) == 0) {
@@ -750,28 +785,26 @@ fu_device_get_specialized_gtype (FuDevice *self)
 	return priv->specialized_gtype;
 }
 
+static void
+fu_device_quirks_iter_cb (FuQuirks *quirks, const gchar *key, const gchar *value, gpointer user_data)
+{
+	FuDevice *self = FU_DEVICE (user_data);
+	g_autoptr(GError) error = NULL;
+	if (!fu_device_set_quirk_kv (self, key, value, &error)) {
+		if (!g_error_matches (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED)) {
+			g_warning ("failed to set quirk key %s=%s: %s",
+				   key, value, error->message);
+		}
+	}
+}
+
 static void
 fu_device_add_guid_quirks (FuDevice *self, const gchar *guid)
 {
 	FuDevicePrivate *priv = GET_PRIVATE (self);
-	const gchar *key;
-	const gchar *value;
-	GHashTableIter iter;
-
-	/* not set */
 	if (priv->quirks == NULL)
 		return;
-	if (!fu_quirks_get_kvs_for_guid (priv->quirks, guid, &iter))
+	fu_quirks_lookup_by_id_iter (priv->quirks, guid, fu_device_quirks_iter_cb, self);
-		return;
-	while (g_hash_table_iter_next (&iter, (gpointer *) &key, (gpointer *) &value)) {
-		g_autoptr(GError) error = NULL;
-		if (!fu_device_set_quirk_kv (self, key, value, &error)) {
-			if (!g_error_matches (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED)) {
-				g_warning ("failed to set quirk key %s=%s: %s",
-					   key, value, error->message);
-			}
-		}
-	}
 }
 
 /**
@@ -2522,6 +2555,7 @@ fu_device_init (FuDevice *self)
 	priv->status = FWUPD_STATUS_IDLE;
 	priv->children = g_ptr_array_new_with_free_func ((GDestroyNotify) g_object_unref);
 	priv->parent_guids = g_ptr_array_new_with_free_func (g_free);
+	priv->possible_plugins = g_ptr_array_new_with_free_func (g_free);
 	g_rw_lock_init (&priv->parent_guids_mutex);
 	priv->metadata = g_hash_table_new_full (g_str_hash, g_str_equal,
 						g_free, g_free);
@@ -2547,6 +2581,7 @@ fu_device_finalize (GObject *object)
 	g_hash_table_unref (priv->metadata);
 	g_ptr_array_unref (priv->children);
 	g_ptr_array_unref (priv->parent_guids);
+	g_ptr_array_unref (priv->possible_plugins);
 	g_free (priv->alternate_id);
 	g_free (priv->equivalent_id);
 	g_free (priv->physical_id);

src/fu-engine.c

@@ -2422,11 +2422,9 @@ fu_engine_load_metadata_store (FuEngine *self, FuEngineLoadFlags flags, GError *
 		xb_builder_import_source (builder, source);
 	}
 
-#if LIBXMLB_CHECK_VERSION(0,1,7)
 	/* on a read-only filesystem don't care about the cache GUID */
 	if (flags & FU_ENGINE_LOAD_FLAG_READONLY_FS)
 		compile_flags |= XB_BUILDER_COMPILE_FLAG_IGNORE_GUID;
-#endif
 
 	/* ensure silo is up to date */
 	cachedirpkg = fu_common_get_path (FU_PATH_KIND_CACHEDIR_PKG);
@@ -3983,9 +3981,9 @@ fu_engine_recoldplug_delay_cb (gpointer user_data)
 static void
 fu_engine_udev_device_add (FuEngine *self, GUdevDevice *udev_device)
 {
-	const gchar *plugin_name;
 	g_autoptr(FuUdevDevice) device = fu_udev_device_new (udev_device);
 	g_autoptr(GError) error_local = NULL;
+	g_autoptr(GPtrArray) possible_plugins = NULL;
 
 	/* add any extra quirks */
 	fu_device_set_quirks (FU_DEVICE (device), self->quirks);
@@ -3997,35 +3995,32 @@ fu_engine_udev_device_add (FuEngine *self, GUdevDevice *udev_device)
 	}
 
 	/* can be specified using a quirk */
-	plugin_name = fu_device_get_plugin (FU_DEVICE (device));
+	possible_plugins = fu_device_get_possible_plugins (FU_DEVICE (device));
-	if (plugin_name != NULL) {
+	for (guint i = 0; i < possible_plugins->len; i++) {
-		g_auto(GStrv) plugins = g_strsplit (plugin_name, ",", -1);
+		FuPlugin *plugin;
-		for (guint i = 0; plugins[i] != NULL; i++) {
+		const gchar *plugin_name = g_ptr_array_index (possible_plugins, i);
-			FuPlugin *plugin;
+		g_autoptr(GError) error = NULL;
-			g_autoptr(GError) error = NULL;
 
-			plugin = fu_plugin_list_find_by_name (self->plugin_list,
+		plugin = fu_plugin_list_find_by_name (self->plugin_list,
-							      plugins[i], &error);
+						      plugin_name, &error);
-			if (plugin == NULL) {
+		if (plugin == NULL) {
-				g_warning ("failed to find specified plugin %s: %s",
+			g_warning ("failed to find specified plugin %s: %s",
-					   plugin_name, error->message);
+				   plugin_name, error->message);
-				continue;
+			continue;
-			}
+		}
-			if (!fu_plugin_runner_udev_device_added (plugin, device, &error)) {
+		if (!fu_plugin_runner_udev_device_added (plugin, device, &error)) {
-				if (g_error_matches (error, FWUPD_ERROR, FWUPD_ERROR_NOT_SUPPORTED)) {
+			if (g_error_matches (error, FWUPD_ERROR, FWUPD_ERROR_NOT_SUPPORTED)) {
-					if (g_getenv ("FWUPD_PROBE_VERBOSE") != NULL) {
+				if (g_getenv ("FWUPD_PROBE_VERBOSE") != NULL) {
-						g_debug ("%s ignoring: %s",
+					g_debug ("%s ignoring: %s",
-							 fu_plugin_get_name (plugin),
+						 fu_plugin_get_name (plugin),
-							 error->message);
+						 error->message);
-					}
-					continue;
 				}
-				g_warning ("failed to add udev device %s: %s",
-					   g_udev_device_get_sysfs_path (udev_device),
-					   error->message);
 				continue;
 			}
-			return;
+			g_warning ("failed to add udev device %s: %s",
+				   g_udev_device_get_sysfs_path (udev_device),
+				   error->message);
+			continue;
 		}
 	}
 }
@@ -4467,10 +4462,10 @@ fu_engine_usb_device_added_cb (GUsbContext *ctx,
 
 
 static void
-fu_engine_load_quirks (FuEngine *self)
+fu_engine_load_quirks (FuEngine *self, FuQuirksLoadFlags quirks_flags)
 {
 	g_autoptr(GError) error = NULL;
-	if (!fu_quirks_load (self->quirks, &error))
+	if (!fu_quirks_load (self->quirks, quirks_flags, &error))
 		g_warning ("Failed to load quirks: %s", error->message);
 }
 
@@ -4655,6 +4650,7 @@ gboolean
 fu_engine_load (FuEngine *self, FuEngineLoadFlags flags, GError **error)
 {
 	FuConfigLoadFlags config_flags = FU_CONFIG_LOAD_FLAG_NONE;
+	FuQuirksLoadFlags quirks_flags = FU_QUIRKS_LOAD_FLAG_NONE;
 	g_autoptr(GPtrArray) checksums = NULL;
 
 	g_return_val_if_fail (FU_IS_ENGINE (self), FALSE);
@@ -4703,7 +4699,10 @@ fu_engine_load (FuEngine *self, FuEngineLoadFlags flags, GError **error)
 	/* load quirks, SMBIOS and the hwids */
 	fu_engine_load_smbios (self);
 	fu_engine_load_hwids (self);
-	fu_engine_load_quirks (self);
+	/* on a read-only filesystem don't care about the cache GUID */
+	if (flags & FU_ENGINE_LOAD_FLAG_READONLY_FS)
+		quirks_flags |= FU_QUIRKS_LOAD_FLAG_READONLY_FS;
+	fu_engine_load_quirks (self, quirks_flags);
 
 	/* load AppStream metadata */
 	if (!fu_engine_load_metadata_store (self, flags, error)) {

src/fu-quirks.c

@@ -11,6 +11,7 @@
 #include <glib-object.h>
 #include <gio/gio.h>
 #include <string.h>
+#include <xmlb.h>
 
 #include "fu-common.h"
 #include "fu-mutex.h"
@@ -50,44 +51,12 @@ static void fu_quirks_finalize	 (GObject *obj);
 struct _FuQuirks
 {
 	GObject			 parent_instance;
-	GPtrArray		*monitors;
+	FuQuirksLoadFlags	 load_flags;
-	GHashTable		*hash;	/* of group:{key:value} */
+	XbSilo			*silo;
-	GRWLock			 hash_mutex;
 };
 
 G_DEFINE_TYPE (FuQuirks, fu_quirks, G_TYPE_OBJECT)
 
-static void
-fu_quirks_monitor_changed_cb (GFileMonitor *monitor,
-			      GFile *file,
-			      GFile *other_file,
-			      GFileMonitorEvent event_type,
-			      gpointer user_data)
-{
-	FuQuirks *self = FU_QUIRKS (user_data);
-	g_autoptr(GError) error = NULL;
-	g_autofree gchar *filename = g_file_get_path (file);
-	g_debug ("%s changed, reloading all configs", filename);
-	if (!fu_quirks_load (self, &error))
-		g_warning ("failed to rescan quirks: %s", error->message);
-}
-
-static gboolean
-fu_quirks_add_inotify (FuQuirks *self, const gchar *filename, GError **error)
-{
-	GFileMonitor *monitor;
-	g_autoptr(GFile) file = g_file_new_for_path (filename);
-
-	/* set up a notify watch */
-	monitor = g_file_monitor (file, G_FILE_MONITOR_NONE, NULL, error);
-	if (monitor == NULL)
-		return FALSE;
-	g_signal_connect (monitor, "changed",
-			  G_CALLBACK (fu_quirks_monitor_changed_cb), self);
-	g_ptr_array_add (self->monitors, monitor);
-	return TRUE;
-}
-
 static gchar *
 fu_quirks_build_group_key (const gchar *group)
 {
@@ -107,158 +76,58 @@ fu_quirks_build_group_key (const gchar *group)
 	return g_strdup (group);
 }
 
-/**
+static GInputStream *
- * fu_quirks_lookup_by_id:
+fu_quirks_convert_quirk_to_xml_cb (XbBuilderSource *self,
- * @self: A #FuPlugin
+				   XbBuilderSourceCtx *ctx,
- * @group: A string group, e.g. "DeviceInstanceId=USB\VID_1235&PID_AB11"
+				   gpointer user_data,
- * @key: An ID to match the entry, e.g. "Name"
+				   GCancellable *cancellable,
- *
+				   GError **error)
- * Looks up an entry in the hardware database using a string value.
- *
- * Returns: (transfer none): values from the database, or %NULL if not found
- *
- * Since: 1.0.1
- **/
-const gchar *
-fu_quirks_lookup_by_id (FuQuirks *self, const gchar *group, const gchar *key)
 {
-	GHashTable *kvs;
+	g_autofree gchar *xml = NULL;
-	g_autofree gchar *group_key = NULL;
-	g_autoptr(GRWLockReaderLocker) locker = g_rw_lock_reader_locker_new (&self->hash_mutex);
-
-	g_return_val_if_fail (FU_IS_QUIRKS (self), NULL);
-	g_return_val_if_fail (group != NULL, NULL);
-	g_return_val_if_fail (key != NULL, NULL);
-	g_return_val_if_fail (locker != NULL, NULL);
-
-	group_key = fu_quirks_build_group_key (group);
-	kvs = g_hash_table_lookup (self->hash, group_key);
-	if (kvs == NULL)
-		return NULL;
-	return g_hash_table_lookup (kvs, key);
-}
-
-/**
- * fu_quirks_get_kvs_for_guid:
- * @self: A #FuPlugin
- * @guid: a GUID
- * @iter: A #GHashTableIter, typically allocated on the stack by the caller
- *
- * Looks up all entries in the hardware database using a GUID value.
- *
- * Returns: %TRUE if the GUID was found, and @iter was set
- *
- * Since: 1.1.2
- **/
-gboolean
-fu_quirks_get_kvs_for_guid (FuQuirks *self, const gchar *guid, GHashTableIter *iter)
-{
-	GHashTable *kvs;
-	g_autoptr(GRWLockReaderLocker) locker = g_rw_lock_reader_locker_new (&self->hash_mutex);
-	g_return_val_if_fail (locker != NULL, FALSE);
-	kvs = g_hash_table_lookup (self->hash, guid);
-	if (kvs == NULL)
-		return FALSE;
-	g_hash_table_iter_init (iter, kvs);
-	return TRUE;
-}
-
-static gchar *
-fu_quirks_merge_values (const gchar *old, const gchar *new)
-{
-	guint cnt = 0;
-	g_autofree gchar **resv = NULL;
-	g_auto(GStrv) newv = g_strsplit (new, ",", -1);
-	g_auto(GStrv) oldv = g_strsplit (old, ",", -1);
-
-	/* segment flags, and append if they do not already exists */
-	resv = g_new0 (gchar *, g_strv_length (oldv) + g_strv_length (newv) + 1);
-	for (guint i = 0; oldv[i] != NULL; i++) {
-		if (!g_strv_contains ((const gchar * const *) resv, oldv[i]))
-			resv[cnt++] = oldv[i];
-	}
-	for (guint i = 0; newv[i] != NULL; i++) {
-		if (!g_strv_contains ((const gchar * const *) resv, newv[i]))
-			resv[cnt++] = newv[i];
-	}
-	return g_strjoinv (",", resv);
-}
-
-/**
- * fu_quirks_add_value: (skip)
- * @self: A #FuQuirks
- * @group: group, e.g. `DeviceInstanceId=USB\VID_0BDA&PID_1100`
- * @key: group, e.g. `Name`
- * @value: group, e.g. `Unknown Device`
- *
- * Adds a value to the quirk database. Normally this is achieved by loading a
- * quirk file using fu_quirks_load().
- *
- * Since: 1.1.2
- **/
-void
-fu_quirks_add_value (FuQuirks *self, const gchar *group, const gchar *key, const gchar *value)
-{
-	GHashTable *kvs;
-	const gchar *value_old;
-	g_autofree gchar *group_key = NULL;
-	g_autofree gchar *value_new = NULL;
-	g_autoptr(GRWLockReaderLocker) locker = g_rw_lock_writer_locker_new (&self->hash_mutex);
-
-	g_return_if_fail (locker != NULL);
-
-	/* does the key already exists in our hash */
-	group_key = fu_quirks_build_group_key (group);
-	kvs = g_hash_table_lookup (self->hash, group_key);
-	if (kvs == NULL) {
-		kvs = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free);
-		g_hash_table_insert (self->hash,
-				     g_steal_pointer (&group_key),
-				     kvs);
-		value_new = g_strdup (value);
-	} else {
-		/* look up in the 2nd level hash */
-		value_old = g_hash_table_lookup (kvs, key);
-		if (value_old != NULL) {
-			g_debug ("already found %s=%s, merging with %s",
-				 group_key, value_old, value);
-			value_new = fu_quirks_merge_values (value_old, value);
-		} else {
-			value_new = g_strdup (value);
-		}
-	}
-
-	/* insert the new value */
-	g_hash_table_insert (kvs, g_strdup (key), g_steal_pointer (&value_new));
-}
-
-static gboolean
-fu_quirks_add_quirks_from_filename (FuQuirks *self, const gchar *filename, GError **error)
-{
-	g_autoptr(GKeyFile) kf = g_key_file_new ();
 	g_auto(GStrv) groups = NULL;
+	g_autoptr(GBytes) bytes = NULL;
+	g_autoptr(GKeyFile) kf = g_key_file_new ();
+	g_autoptr(XbBuilderNode) root = xb_builder_node_new ("quirk");
 
-	/* load keyfile */
+	/* parse keyfile */
-	if (!g_key_file_load_from_file (kf, filename, G_KEY_FILE_NONE, error))
+	bytes = xb_builder_source_ctx_get_bytes (ctx, cancellable, error);
-		return FALSE;
+	if (bytes == NULL)
+		return NULL;
+	if (!g_key_file_load_from_data (kf,
+					g_bytes_get_data (bytes, NULL),
+					g_bytes_get_size (bytes),
+					G_KEY_FILE_NONE,
+					error))
+		return NULL;
 
 	/* add each set of groups and keys */
 	groups = g_key_file_get_groups (kf, NULL);
 	for (guint i = 0; groups[i] != NULL; i++) {
 		g_auto(GStrv) keys = NULL;
+		g_autofree gchar *group_id = NULL;
+		g_autoptr(XbBuilderNode) bn = NULL;
 		keys = g_key_file_get_keys (kf, groups[i], NULL, error);
 		if (keys == NULL)
 			return FALSE;
+		group_id = fu_quirks_build_group_key (groups[i]);
+		bn = xb_builder_node_insert (root, "device", "id", group_id, NULL);
 		for (guint j = 0; keys[j] != NULL; j++) {
 			g_autofree gchar *value = NULL;
-			/* get value from keyfile */
 			value = g_key_file_get_value (kf, groups[i], keys[j], error);
 			if (value == NULL)
-				return FALSE;
+				return NULL;
-			fu_quirks_add_value (self, groups[i], keys[j], value);
+			xb_builder_node_insert_text (bn,
+						     "value", value,
+						     "key", keys[j],
+						     NULL);
 		}
 	}
-	return TRUE;
+
+	/* export as XML */
+	xml = xb_builder_node_export (root, XB_NODE_EXPORT_FLAG_ADD_HEADER, error);
+	if (xml == NULL)
+		return NULL;
+	return g_memory_input_stream_new_from_data (g_steal_pointer (&xml), -1, g_free);
 }
 
 static gint
@@ -270,7 +139,8 @@ fu_quirks_filename_sort_cb (gconstpointer a, gconstpointer b)
 }
 
 static gboolean
-fu_quirks_add_quirks_for_path (FuQuirks *self, const gchar *path, GError **error)
+fu_quirks_add_quirks_for_path (FuQuirks *self, XbBuilder *builder,
+			       const gchar *path, GError **error)
 {
 	const gchar *tmp;
 	g_autofree gchar *path_hw = NULL;
@@ -300,21 +170,200 @@ fu_quirks_add_quirks_for_path (FuQuirks *self, const gchar *path, GError **error
 	/* process files */
 	for (guint i = 0; i < filenames->len; i++) {
 		const gchar *filename = g_ptr_array_index (filenames, i);
+		g_autoptr(GFile) file = g_file_new_for_path (filename);
+		g_autoptr(XbBuilderSource) source = xb_builder_source_new ();
 
 		/* load from keyfile */
-		g_debug ("loading quirks from %s", filename);
+		xb_builder_source_add_adapter (source, "text/plain",
-		if (!fu_quirks_add_quirks_from_filename (self, filename, error)) {
+					       fu_quirks_convert_quirk_to_xml_cb,
+					       NULL, NULL);
+		if (!xb_builder_source_load_file (source, file,
+						  XB_BUILDER_SOURCE_FLAG_WATCH_FILE |
+						  XB_BUILDER_SOURCE_FLAG_LITERAL_TEXT,
+						  NULL, error)) {
 			g_prefix_error (error, "failed to load %s: ", filename);
 			return FALSE;
 		}
 
 		/* watch the file for changes */
-		if (!fu_quirks_add_inotify (self, filename, error))
+		xb_builder_import_source (builder, source);
-			return FALSE;
 	}
 
 	/* success */
-	g_debug ("now %u quirk entries", g_hash_table_size (self->hash));
+	return TRUE;
+}
+
+static gboolean
+fu_quirks_check_silo (FuQuirks *self, GError **error)
+{
+	XbBuilderCompileFlags compile_flags = XB_BUILDER_COMPILE_FLAG_WATCH_BLOB;
+	g_autofree gchar *cachedirpkg = NULL;
+	g_autofree gchar *datadir = NULL;
+	g_autofree gchar *localstatedir = NULL;
+	g_autofree gchar *xmlbfn = NULL;
+	g_autoptr(GFile) file = NULL;
+	g_autoptr(XbBuilder) builder = NULL;
+
+	/* everything is okay */
+	if (self->silo != NULL && xb_silo_is_valid (self->silo))
+		return TRUE;
+
+	/* system datadir */
+	builder = xb_builder_new ();
+	datadir = fu_common_get_path (FU_PATH_KIND_DATADIR_PKG);
+	if (!fu_quirks_add_quirks_for_path (self, builder, datadir, error))
+		return FALSE;
+
+	/* something we can write when using Ostree */
+	localstatedir = fu_common_get_path (FU_PATH_KIND_LOCALSTATEDIR_PKG);
+	if (!fu_quirks_add_quirks_for_path (self, builder, localstatedir, error))
+		return FALSE;
+
+	/* load silo */
+	cachedirpkg = fu_common_get_path (FU_PATH_KIND_CACHEDIR_PKG);
+	xmlbfn = g_build_filename (cachedirpkg, "quirks.xmlb", NULL);
+	file = g_file_new_for_path (xmlbfn);
+	if (g_getenv ("XMLB_VERBOSE") != NULL) {
+		xb_builder_set_profile_flags (builder,
+					      XB_SILO_PROFILE_FLAG_XPATH |
+					      XB_SILO_PROFILE_FLAG_DEBUG);
+	}
+	if (self->load_flags & FU_QUIRKS_LOAD_FLAG_READONLY_FS)
+		compile_flags |= XB_BUILDER_COMPILE_FLAG_IGNORE_GUID;
+	self->silo = xb_builder_ensure (builder, file, compile_flags, NULL, error);
+	return self->silo != NULL;
+}
+
+/**
+ * fu_quirks_lookup_by_id:
+ * @self: A #FuPlugin
+ * @group: A string group, e.g. "DeviceInstanceId=USB\VID_1235&PID_AB11"
+ * @key: An ID to match the entry, e.g. "Name"
+ *
+ * Looks up an entry in the hardware database using a string value.
+ *
+ * Returns: (transfer none): values from the database, or %NULL if not found
+ *
+ * Since: 1.0.1
+ **/
+const gchar *
+fu_quirks_lookup_by_id (FuQuirks *self, const gchar *group, const gchar *key)
+{
+	g_autofree gchar *group_key = NULL;
+	g_autoptr(GError) error = NULL;
+	g_autoptr(XbNode) n = NULL;
+	g_autoptr(XbQuery) query = NULL;
+
+	g_return_val_if_fail (FU_IS_QUIRKS (self), NULL);
+	g_return_val_if_fail (group != NULL, NULL);
+	g_return_val_if_fail (key != NULL, NULL);
+
+	/* ensure up to date */
+	if (!fu_quirks_check_silo (self, &error)) {
+		g_warning ("failed to build silo: %s", error->message);
+		return NULL;
+	}
+
+	/* query */
+	group_key = fu_quirks_build_group_key (group);
+	query = xb_query_new_full (self->silo,
+				   "quirk/device[@id=?]/value[@key=?]",
+				   XB_QUERY_FLAG_NONE,
+				   &error);
+	if (query == NULL) {
+		if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+			return NULL;
+		if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT))
+			return NULL;
+		g_warning ("failed to build query: %s", error->message);
+		return NULL;
+	}
+	if (!xb_query_bind_str (query, 0, group_key, &error)) {
+		g_warning ("failed to bind 0: %s", error->message);
+		return NULL;
+	}
+	if (!xb_query_bind_str (query, 1, key, &error)) {
+		g_warning ("failed to bind 1: %s", error->message);
+		return NULL;
+	}
+	n = xb_silo_query_first_full (self->silo, query, &error);
+	if (n == NULL) {
+		if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+			return NULL;
+		if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT))
+			return NULL;
+		g_warning ("failed to query: %s", error->message);
+		return NULL;
+	}
+	return xb_node_get_text (n);
+}
+
+/**
+ * fu_quirks_lookup_by_id_iter:
+ * @self: A #FuPlugin
+ * @guid: a GUID
+ * @iter_cb: A #FuQuirksIter
+ * @user_data: user data passed to @iter_cb
+ *
+ * Looks up all entries in the hardware database using a GUID value.
+ *
+ * Returns: %TRUE if the ID was found, and @iter was called
+ *
+ * Since: 1.3.3
+ **/
+gboolean
+fu_quirks_lookup_by_id_iter (FuQuirks *self, const gchar *group,
+			     FuQuirksIter iter_cb, gpointer user_data)
+{
+	g_autofree gchar *group_key = NULL;
+	g_autoptr(GError) error = NULL;
+	g_autoptr(GPtrArray) results = NULL;
+	g_autoptr(XbQuery) query = NULL;
+
+	g_return_val_if_fail (FU_IS_QUIRKS (self), FALSE);
+	g_return_val_if_fail (group != NULL, FALSE);
+	g_return_val_if_fail (iter_cb != NULL, FALSE);
+
+	/* ensure up to date */
+	if (!fu_quirks_check_silo (self, &error)) {
+		g_warning ("failed to build silo: %s", error->message);
+		return FALSE;
+	}
+
+	/* query */
+	group_key = fu_quirks_build_group_key (group);
+	query = xb_query_new_full (self->silo,
+				   "quirk/device[@id=?]/value",
+				   XB_QUERY_FLAG_NONE,
+				   &error);
+	if (query == NULL) {
+		if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+			return FALSE;
+		if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT))
+			return FALSE;
+		g_warning ("failed to build query: %s", error->message);
+		return FALSE;
+	}
+	if (!xb_query_bind_str (query, 0, group_key, &error)) {
+		g_warning ("failed to bind 0: %s", error->message);
+		return FALSE;
+	}
+	results = xb_silo_query_full (self->silo, query, &error);
+	if (results == NULL) {
+		if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+			return FALSE;
+		if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT))
+			return FALSE;
+		g_warning ("failed to query: %s", error->message);
+		return FALSE;
+	}
+	for (guint i = 0; i < results->len; i++) {
+		XbNode *n = g_ptr_array_index (results, i);
+		iter_cb (self,
+			 xb_node_get_attr (n, "key"),
+			 xb_node_get_text (n),
+			 user_data);
+	}
 	return TRUE;
 }
 
@@ -330,31 +379,11 @@ fu_quirks_add_quirks_for_path (FuQuirks *self, const gchar *path, GError **error
  * Since: 1.0.1
  **/
 gboolean
-fu_quirks_load (FuQuirks *self, GError **error)
+fu_quirks_load (FuQuirks *self, FuQuirksLoadFlags load_flags, GError **error)
 {
-	g_autofree gchar *datadir = NULL;
-	g_autofree gchar *localstatedir = NULL;
-
 	g_return_val_if_fail (FU_IS_QUIRKS (self), FALSE);
-
+	self->load_flags = load_flags;
-	/* ensure empty in case we're called from a monitor change */
+	return fu_quirks_check_silo (self, error);
-	g_ptr_array_set_size (self->monitors, 0);
-	g_rw_lock_writer_lock (&self->hash_mutex);
-	g_hash_table_remove_all (self->hash);
-	g_rw_lock_writer_unlock (&self->hash_mutex);
-
-	/* system datadir */
-	datadir = fu_common_get_path (FU_PATH_KIND_DATADIR_PKG);
-	if (!fu_quirks_add_quirks_for_path (self, datadir, error))
-		return FALSE;
-
-	/* something we can write when using Ostree */
-	localstatedir = fu_common_get_path (FU_PATH_KIND_LOCALSTATEDIR_PKG);
-	if (!fu_quirks_add_quirks_for_path (self, localstatedir, error))
-		return FALSE;
-
-	/* success */
-	return TRUE;
 }
 
 static void
@@ -367,18 +396,14 @@ fu_quirks_class_init (FuQuirksClass *klass)
 static void
 fu_quirks_init (FuQuirks *self)
 {
-	self->monitors = g_ptr_array_new_with_free_func ((GDestroyNotify) g_object_unref);
-	self->hash = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify) g_hash_table_unref);
-	g_rw_lock_init (&self->hash_mutex);
 }
 
 static void
 fu_quirks_finalize (GObject *obj)
 {
 	FuQuirks *self = FU_QUIRKS (obj);
-	g_ptr_array_unref (self->monitors);
+	if (self->silo != NULL)
-	g_rw_lock_clear (&self->hash_mutex);
+		g_object_unref (self->silo);
-	g_hash_table_unref (self->hash);
 	G_OBJECT_CLASS (fu_quirks_parent_class)->finalize (obj);
 }
 

src/fu-quirks.h

@@ -12,19 +12,36 @@
 #define FU_TYPE_QUIRKS (fu_quirks_get_type ())
 G_DECLARE_FINAL_TYPE (FuQuirks, fu_quirks, FU, QUIRKS, GObject)
 
+/**
+ * FuQuirksLoadFlags:
+ * @FU_QUIRKS_LOAD_FLAG_NONE:		No flags set
+ * @FU_QUIRKS_LOAD_FLAG_READONLY_FS:	Ignore readonly filesystem errors
+ *
+ * The flags to use when loading quirks.
+ **/
+typedef enum {
+	FU_QUIRKS_LOAD_FLAG_NONE		= 0,
+	FU_QUIRKS_LOAD_FLAG_READONLY_FS		= 1 << 0,
+	/*< private >*/
+	FU_QUIRKS_LOAD_FLAG_LAST
+} FuQuirksLoadFlags;
+
+typedef void	(*FuQuirksIter)				(FuQuirks	*self,
+							 const gchar	*key,
+							 const gchar	*value,
+							 gpointer	 user_data);
+
 FuQuirks	*fu_quirks_new				(void);
 gboolean	 fu_quirks_load				(FuQuirks	*self,
+							 FuQuirksLoadFlags load_flags,
 							 GError		**error);
 const gchar	*fu_quirks_lookup_by_id			(FuQuirks	*self,
 							 const gchar	*group,
 							 const gchar	*key);
-void		 fu_quirks_add_value			(FuQuirks	*self,
+gboolean	 fu_quirks_lookup_by_id_iter		(FuQuirks	*self,
 							 const gchar	*group,
-							 const gchar	*key,
+							 FuQuirksIter	 iter,
-							 const gchar	*value);
+							 gpointer	 user_data);
-gboolean	 fu_quirks_get_kvs_for_guid		(FuQuirks	*self,
-							 const gchar	*guid,
-							 GHashTableIter *iter);
 
 #define	FU_QUIRKS_PLUGIN			"Plugin"
 #define	FU_QUIRKS_UEFI_VERSION_FORMAT		"UefiVersionFormat"

src/fu-self-test.c

@@ -2114,20 +2114,18 @@ fu_plugin_quirks_func (void)
 	g_autoptr(FuPlugin) plugin = fu_plugin_new ();
 	g_autoptr(GError) error = NULL;
 
-	ret = fu_quirks_load (quirks, &error);
+	ret = fu_quirks_load (quirks, FU_QUIRKS_LOAD_FLAG_NONE, &error);
 	g_assert_no_error (error);
 	g_assert (ret);
 	fu_plugin_set_quirks (plugin, quirks);
 
 	/* exact */
 	tmp = fu_plugin_lookup_quirk_by_id (plugin, "USB\\VID_0A5C&PID_6412", "Flags");
-	g_assert_cmpstr (tmp, ==, "MERGE_ME,ignore-runtime");
+	g_assert_cmpstr (tmp, ==, "ignore-runtime");
 	tmp = fu_plugin_lookup_quirk_by_id (plugin, "ACME Inc.=True", "Test");
 	g_assert_cmpstr (tmp, ==, "awesome");
 	tmp = fu_plugin_lookup_quirk_by_id (plugin, "CORP*", "Test");
 	g_assert_cmpstr (tmp, ==, "town");
-	tmp = fu_plugin_lookup_quirk_by_id (plugin, "USB\\VID_FFFF&PID_FFFF", "Flags");
-	g_assert_cmpstr (tmp, ==, "");
 	tmp = fu_plugin_lookup_quirk_by_id (plugin, "baz", "Unfound");
 	g_assert_cmpstr (tmp, ==, NULL);
 	tmp = fu_plugin_lookup_quirk_by_id (plugin, "unfound", "tests");
@@ -2141,29 +2139,23 @@ fu_plugin_quirks_func (void)
 static void
 fu_plugin_quirks_performance_func (void)
 {
+	gboolean ret;
 	g_autoptr(FuQuirks) quirks = fu_quirks_new ();
 	g_autoptr(GTimer) timer = g_timer_new ();
-	const gchar *keys[] = {
+	g_autoptr(GError) error = NULL;
-		"Name", "Icon", "Children", "Plugin", "Flags",
+	const gchar *keys[] = { "Name", "Children", "Flags", NULL };
-		"FirmwareSizeMin", "FirmwareSizeMax", NULL };
 
-	/* insert */
+	ret = fu_quirks_load (quirks, FU_QUIRKS_LOAD_FLAG_NONE, &error);
-	for (guint j = 0; j < 1000; j++) {
+	g_assert_no_error (error);
-		g_autofree gchar *group = NULL;
+	g_assert (ret);
-		group = g_strdup_printf ("DeviceInstanceId=USB\\VID_0BDA&PID_%04X", j);
-		for (guint i = 0; keys[i] != NULL; i++)
-			fu_quirks_add_value (quirks, group, keys[i], "Value");
-	}
-	g_print ("insert=%.3fms ", g_timer_elapsed (timer, NULL) * 1000.f);
 
 	/* lookup */
 	g_timer_reset (timer);
 	for (guint j = 0; j < 1000; j++) {
-		g_autofree gchar *group = NULL;
+		const gchar *group = "DeviceInstanceId=USB\\VID_0BDA&PID_1100";
-		group = g_strdup_printf ("DeviceInstanceId=USB\\VID_0BDA&PID_%04X", j);
 		for (guint i = 0; keys[i] != NULL; i++) {
 			const gchar *tmp = fu_quirks_lookup_by_id (quirks, group, keys[i]);
-			g_assert_cmpstr (tmp, ==, "Value");
+			g_assert_cmpstr (tmp, !=, NULL);
 		}
 	}
 	g_print ("lookup=%.3fms ", g_timer_elapsed (timer, NULL) * 1000.f);
@@ -2179,7 +2171,7 @@ fu_plugin_quirks_device_func (void)
 	g_autoptr(FuQuirks) quirks = fu_quirks_new ();
 	g_autoptr(GError) error = NULL;
 
-	ret = fu_quirks_load (quirks, &error);
+	ret = fu_quirks_load (quirks, FU_QUIRKS_LOAD_FLAG_NONE, &error);
 	g_assert_no_error (error);
 	g_assert (ret);
 

subprojects/libxmlb.wrap

@@ -1,4 +1,4 @@
 [wrap-git]
 directory = libxmlb
 url = https://github.com/hughsie/libxmlb.git
-revision = 0.1.7
+revision = 0.1.13