diff --git a/panels/keyboard/cc-keyboard-shortcut-editor.c b/panels/keyboard/cc-keyboard-shortcut-editor.c
index c09686c03..efb2bc80a 100644
--- a/panels/keyboard/cc-keyboard-shortcut-editor.c
+++ b/panels/keyboard/cc-keyboard-shortcut-editor.c
@@ -319,6 +319,7 @@ set_header_mode (CcKeyboardShortcutEditor *self,
                                                mode != HEADER_MODE_CUSTOM_EDIT);
   gtk_widget_set_visible (self->replace_button, mode == HEADER_MODE_REPLACE);
   gtk_widget_set_visible (self->set_button, mode == HEADER_MODE_SET);
+  gtk_widget_set_visible (self->remove_button, mode == HEADER_MODE_CUSTOM_EDIT);
 
   /* By setting the default response, the action button gets the 'suggested-action' applied */
   switch (mode)
@@ -643,7 +644,6 @@ setup_keyboard_item (CcKeyboardShortcutEditor *self,
                                         is_accel_empty ? "button" : "label");
 
       gtk_widget_set_visible (self->reset_custom_button, !is_accel_empty);
-      gtk_widget_show (self->remove_button);
 
       g_signal_handlers_unblock_by_func (self->command_entry, command_entry_changed_cb, self);
       g_signal_handlers_unblock_by_func (self->name_entry, name_entry_changed_cb, self);
@@ -1040,6 +1040,5 @@ cc_keyboard_shortcut_editor_set_mode (CcKeyboardShortcutEditor *self,
       gtk_widget_set_sensitive (self->add_button, FALSE);
 
       gtk_widget_hide (self->reset_custom_button);
-      gtk_widget_hide (self->remove_button);
     }
 }
diff --git a/panels/keyboard/shortcut-editor.ui b/panels/keyboard/shortcut-editor.ui
index 4cd455854..90100e72a 100644
--- a/panels/keyboard/shortcut-editor.ui
+++ b/panels/keyboard/shortcut-editor.ui
@@ -280,22 +280,6 @@
                     <property name="top_attach">2</property>
                   </packing>
                 </child>
-                <child>
-                  <object class="GtkButton" id="remove_button">
-                    <property name="visible">True</property>
-                    <property name="can_focus">False</property>
-                    <property name="label" translatable="yes">Remove</property>
-                    <property name="valign">end</property>
-                    <signal name="clicked" handler="remove_button_clicked_cb" object="CcKeyboardShortcutEditor" swapped="yes" />
-                    <style>
-                      <class name="destructive-action" />
-                    </style>
-                  </object>
-                  <packing>
-                    <property name="left_attach">2</property>
-                    <property name="top_attach">3</property>
-                  </packing>
-                </child>
               </object>
               <packing>
                 <property name="name">custom</property>
@@ -364,6 +348,18 @@
             <signal name="clicked" handler="cancel_button_clicked_cb" object="CcKeyboardShortcutEditor" swapped="no" />
           </object>
         </child>
+        <child>
+          <object class="GtkButton" id="remove_button">
+            <property name="visible">True</property>
+            <property name="can_focus">False</property>
+            <property name="label" translatable="yes">Remove</property>
+            <property name="valign">end</property>
+            <signal name="clicked" handler="remove_button_clicked_cb" object="CcKeyboardShortcutEditor" swapped="yes" />
+            <style>
+              <class name="destructive-action" />
+            </style>
+          </object>
+        </child>
         <child>
           <object class="GtkButton" id="add_button">
             <property name="label" translatable="yes">Add</property>