diff --git a/panels/system/users/cc-avatar-chooser.c b/panels/system/users/cc-avatar-chooser.c
index 75dd06315..fcb75287d 100644
--- a/panels/system/users/cc-avatar-chooser.c
+++ b/panels/system/users/cc-avatar-chooser.c
@@ -72,7 +72,7 @@ crop_dialog_response (CcAvatarChooser *self,
         pb2 = gdk_pixbuf_scale_simple (pb, AVATAR_PIXEL_SIZE, AVATAR_PIXEL_SIZE, GDK_INTERP_BILINEAR);
         texture = gdk_texture_new_for_pixbuf (pb2);
 
-        set_user_icon_data (self->user, texture);
+        set_user_icon_data (self->user, texture, IMAGE_SOURCE_VALUE_CUSTOM);
 
         self->crop_area = NULL;
         gtk_window_destroy (GTK_WINDOW (dialog));
@@ -187,11 +187,20 @@ face_widget_activated (CcAvatarChooser *self,
         const gchar *filename;
         GtkWidget   *image;
         g_autoptr(GdkTexture) texture = NULL;
+        g_autoptr(GError) error = NULL;
 
         image = gtk_flow_box_child_get_child (child);
         filename = g_object_get_data (G_OBJECT (image), "filename");
 
-        act_user_set_icon_file (self->user, filename);
+        if (filename != NULL) {
+                texture = gdk_texture_new_from_filename (filename, &error);
+        }
+
+        if (error != NULL) {
+                g_warning ("Failed to load selected avatar image: %s", error->message);
+        } else {
+                set_user_icon_data (self->user, texture, IMAGE_SOURCE_VALUE_FACE);
+        }
 
         gtk_popover_popdown (GTK_POPOVER (self));
 }
diff --git a/panels/system/users/user-utils.c b/panels/system/users/user-utils.c
index faecdac0f..e6218085d 100644
--- a/panels/system/users/user-utils.c
+++ b/panels/system/users/user-utils.c
@@ -311,8 +311,9 @@ draw_avatar_to_texture (AdwAvatar *avatar, int size)
 }
 
 void
-set_user_icon_data (ActUser   *user,
-                    GdkTexture *texture)
+set_user_icon_data (ActUser     *user,
+                    GdkTexture  *texture,
+                    const gchar *image_source)
 {
         g_autofree gchar *path = NULL;
         g_autoptr(GError) error = NULL;
@@ -325,7 +326,13 @@ set_user_icon_data (ActUser   *user,
                 return;
         }
 
-        gdk_texture_save_to_png (texture, path);
+        g_autoptr(GdkPixbuf) pixbuf = gdk_pixbuf_get_from_texture (texture);
+        gdk_pixbuf_save (pixbuf, path, "png", &error, IMAGE_SOURCE_KEY, image_source, NULL);
+
+        if (error != NULL) {
+            g_warning ("Failed to create temporary user icon: %s", error->message);
+        }
+
         close (fd);
 
         act_user_set_icon_file (user, path);
@@ -359,12 +366,26 @@ setup_avatar_for_user (AdwAvatar *avatar, ActUser *user)
         avatar_file = act_user_get_icon_file (user);
         if (avatar_file) {
                 g_autoptr(GdkPixbuf) pixbuf = NULL;
+                const gchar *image_source;
+                gboolean is_generated = TRUE;
 
                 pixbuf = gdk_pixbuf_new_from_file_at_size (avatar_file,
                                                            adw_avatar_get_size (avatar),
                                                            adw_avatar_get_size (avatar),
                                                            NULL);
+
                 if (pixbuf) {
+                        image_source = gdk_pixbuf_get_option (pixbuf, IMAGE_SOURCE_KEY);
+
+                        if (image_source == NULL)
+                                g_debug ("User avatar's source isn't defined");
+                        else
+                                g_debug ("User avatar's source is %s", image_source);
+
+                        is_generated = g_strcmp0 (image_source, "gnome-generated") == 0;
+                }
+
+                if (!is_generated) {
                         g_autoptr(GdkTexture) texture = NULL;
 
                         texture = gdk_texture_new_for_pixbuf (pixbuf);
diff --git a/panels/system/users/user-utils.h b/panels/system/users/user-utils.h
index f88164ace..8851893e2 100644
--- a/panels/system/users/user-utils.h
+++ b/panels/system/users/user-utils.h
@@ -28,6 +28,15 @@ G_BEGIN_DECLS
 
 #define AVATAR_PIXEL_SIZE 512
 
+/* Key and values that are written as metadata to the exported user avatar this
+ * way it's possible to know how the image was initially created.
+ * If set to generated we can regenerated the avatar when the style changes or
+ * when the users full name changes. The other two values don't have a specific use yet */
+#define IMAGE_SOURCE_KEY "tEXt::source"
+#define IMAGE_SOURCE_VALUE_GENERATED "gnome-generated"
+#define IMAGE_SOURCE_VALUE_FACE "gnome-face"
+#define IMAGE_SOURCE_VALUE_CUSTOM "gnome-custom"
+
 void     set_entry_generation_icon        (GtkEntry    *entry);
 void     set_entry_validation_checkmark   (GtkEntry    *entry);
 void     set_entry_validation_error       (GtkEntry    *entry,
@@ -49,7 +58,8 @@ gboolean is_valid_username_finish         (GAsyncResult *result,
 GdkTexture *draw_avatar_to_texture        (AdwAvatar *avatar,
                                            int        size);
 void       set_user_icon_data             (ActUser     *user,
-                                           GdkTexture  *texture);
+                                           GdkTexture  *texture,
+                                           const gchar *image_source);
 void       setup_avatar_for_user          (AdwAvatar *avatar,
                                            ActUser *user);
 GSettings *settings_or_null               (const gchar *schema);