diff --git a/panels/keyboard/cc-keyboard-item.c b/panels/keyboard/cc-keyboard-item.c
index ec7e1308c..dbadfb319 100644
--- a/panels/keyboard/cc-keyboard-item.c
+++ b/panels/keyboard/cc-keyboard-item.c
@@ -314,10 +314,9 @@ cc_keyboard_item_finalize (GObject *object)
   /* Remove GConf watches */
   client = gconf_client_get_default ();
 
-  /* FIXME what if we didn't add a watch? */
-  if (item->gconf_key_dir != NULL)
+  if (item->gconf_key_dir != NULL && item->monitored_dir)
     gconf_client_remove_dir (client, item->gconf_key_dir, NULL);
-  else if (item->gconf_key != NULL)
+  else if (item->gconf_key != NULL && item->monitored)
     gconf_client_remove_dir (client, item->gconf_key, NULL);
 
   if (item->gconf_cnxn != 0)
@@ -459,6 +458,7 @@ cc_keyboard_item_load_from_gconf (CcKeyboardItem *item,
   }
   item->editable = gconf_entry_get_is_writable (entry);
   gconf_client_add_dir (client, item->gconf_key, GCONF_CLIENT_PRELOAD_ONELEVEL, NULL);
+  item->monitored = TRUE;
   item->gconf_cnxn = gconf_client_notify_add (client,
                                               item->gconf_key,
                                               (GConfClientNotifyFunc) &keybinding_key_changed,
@@ -501,6 +501,7 @@ cc_keyboard_item_load_from_gconf_dir (CcKeyboardItem *item,
   item->command = gconf_client_get_string (client, item->cmd_gconf_key, NULL);
   item->editable = gconf_entry_get_is_writable (entry);
   gconf_client_add_dir (client, item->gconf_key_dir, GCONF_CLIENT_PRELOAD_ONELEVEL, NULL);
+  item->monitored_dir = TRUE;
 
   item->desc_editable = gconf_client_key_is_writable (client, item->desc_gconf_key, NULL);
   item->gconf_cnxn_desc = gconf_client_notify_add (client,
diff --git a/panels/keyboard/cc-keyboard-item.h b/panels/keyboard/cc-keyboard-item.h
index ce290528a..859d4163f 100644
--- a/panels/keyboard/cc-keyboard-item.h
+++ b/panels/keyboard/cc-keyboard-item.h
@@ -72,6 +72,7 @@ typedef struct
   char *gconf_key;
   guint gconf_cnxn;
   gboolean editable;
+  gboolean monitored;
 
   /* GCONF DIR */
   char *gconf_key_dir;
@@ -87,6 +88,8 @@ typedef struct
   gboolean cmd_editable;
   guint gconf_cnxn_cmd;
 
+  gboolean monitored_dir;
+
   /* GSettings */
   char *schema;
   char *key;