diff --git a/common/project/project_local_settings.cpp b/common/project/project_local_settings.cpp
index 21500d3ca0..ee0def38c8 100644
--- a/common/project/project_local_settings.cpp
+++ b/common/project/project_local_settings.cpp
@@ -83,7 +83,8 @@ PROJECT_LOCAL_SETTINGS::PROJECT_LOCAL_SETTINGS( PROJECT* aProject, const wxStrin
                     return;
                 }
 
-                m_VisibleItems.reset();
+                m_VisibleItems &= ~UserVisbilityLayers();
+                GAL_SET visible;
 
                 for( const nlohmann::json& entry : aVal )
                 {
@@ -92,13 +93,15 @@ PROJECT_LOCAL_SETTINGS::PROJECT_LOCAL_SETTINGS( PROJECT* aProject, const wxStrin
                         std::string vs = entry.get<std::string>();
 
                         if( std::optional<GAL_LAYER_ID> l = RenderLayerFromVisbilityString( vs ) )
-                            m_VisibleItems.set( *l );
+                            visible.set( *l );
                     }
                     catch( ... )
                     {
                         // Non-integer or out of range entry in the array; ignore
                     }
                 }
+
+                m_VisibleItems |= UserVisbilityLayers() & visible;
             },
             {} ) );
 
diff --git a/common/settings/layer_settings_utils.cpp b/common/settings/layer_settings_utils.cpp
index 5851c0d749..3902d9818f 100644
--- a/common/settings/layer_settings_utils.cpp
+++ b/common/settings/layer_settings_utils.cpp
@@ -19,9 +19,41 @@
 
 #include <boost/algorithm/string/case_conv.hpp>
 #include <magic_enum.hpp>
+
+#include <core/arraydim.h>
 #include <settings/layer_settings_utils.h>
 
 
+GAL_SET UserVisbilityLayers()
+{
+    static const GAL_LAYER_ID layers[] = {
+        LAYER_TRACKS,
+        LAYER_VIAS,
+        LAYER_PADS,
+        LAYER_ZONES,
+        LAYER_SHAPES,
+        LAYER_DRAW_BITMAPS,
+        LAYER_FOOTPRINTS_FR,
+        LAYER_FOOTPRINTS_BK,
+        LAYER_FP_VALUES,
+        LAYER_FP_REFERENCES,
+        LAYER_FP_TEXT,
+        LAYER_ANCHOR,
+        LAYER_RATSNEST,
+        LAYER_DRC_WARNING,
+        LAYER_DRC_ERROR,
+        LAYER_DRC_EXCLUSION,
+        LAYER_LOCKED_ITEM_SHADOW,
+        LAYER_CONFLICTS_SHADOW,
+        LAYER_DRAWINGSHEET,
+        LAYER_GRID,
+    };
+
+    static const GAL_SET saved( layers, arrayDim( layers ) );
+    return saved;
+}
+
+
 GAL_LAYER_ID RenderLayerFromVisibilityLayer( VISIBILITY_LAYER aLayer )
 {
     switch( aLayer )
diff --git a/include/settings/layer_settings_utils.h b/include/settings/layer_settings_utils.h
index 85addad781..f14e5d1c50 100644
--- a/include/settings/layer_settings_utils.h
+++ b/include/settings/layer_settings_utils.h
@@ -53,6 +53,11 @@ enum class VISIBILITY_LAYER
     GRID
 };
 
+/**
+ * The set of GAL_LAYER_IDs that correspond to VISIBILITY_LAYERS
+ */
+GAL_SET UserVisbilityLayers();
+
 GAL_LAYER_ID RenderLayerFromVisibilityLayer( VISIBILITY_LAYER aLayer );
 std::optional<VISIBILITY_LAYER> VisibilityLayerFromRenderLayer( GAL_LAYER_ID aLayerId );