diff --git a/osu.Game/Extensions/DrawableExtensions.cs b/osu.Game/Extensions/DrawableExtensions.cs
index f66fe2b8b2..1199db587a 100644
--- a/osu.Game/Extensions/DrawableExtensions.cs
+++ b/osu.Game/Extensions/DrawableExtensions.cs
@@ -2,9 +2,6 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System;
-using System.Collections.Generic;
-using System.Runtime.CompilerServices;
-using osu.Framework.Bindables;
 using osu.Framework.Graphics;
 using osu.Framework.Graphics.Containers;
 using osu.Framework.Input.Bindings;
@@ -60,7 +57,9 @@ namespace osu.Game.Extensions
             component.Scale = info.Scale;
             component.Anchor = info.Anchor;
             component.Origin = info.Origin;
-            component.UsingClosestAnchor().Value = info.UsingClosestAnchor;
+
+            if (component is ISkinnableDrawable skinnable)
+                skinnable.UsingClosestAnchor = info.UsingClosestAnchor;
 
             if (component is Container container)
             {
@@ -68,26 +67,5 @@ namespace osu.Game.Extensions
                     container.Add(child.CreateInstance());
             }
         }
-
-        /// <remarks>
-        /// <p>A <see cref="ConditionalWeakTable{TKey,TValue}">ConditionalWeakTable</see> is preferable to a <see cref="Dictionary{TKey,TValue}">Dictionary</see> because a <c>Dictionary</c> will keep
-        /// orphaned references to an <see cref="ISkinnableDrawable"/> forever, unless manually pruned.</p>
-        /// <p><see cref="BindableBool"/> is used as a thin wrapper around <see cref="System.Boolean">bool</see> because <c>ConditionalWeakTable</c> requires a reference type as both a key and a value.</p>
-        /// <p><see cref="IDrawable"/> was chosen over <see cref="Drawable"/> because it is a common ancestor between <see cref="Drawable"/> (which is required for <see cref="Drawable.Anchor"/> logic)
-        /// and <see cref="ISkinnableDrawable"/> (which is required for serialization via <see cref="SkinnableInfo"/>).</p>
-        /// <p>This collection is thread-safe according to the
-        /// <a href="https://docs.microsoft.com/en-us/dotnet/api/system.runtime.compilerservices.conditionalweaktable-2?view=net-5.0#thread-safety">documentation</a>,
-        /// but the <c>BindableBool</c>s are not unless <see cref="Bindable{T}.BeginLease">leased</see>.</p>
-        /// </remarks>
-        private static readonly ConditionalWeakTable<IDrawable, BindableBool> is_drawable_using_closest_anchor_lookup = new ConditionalWeakTable<IDrawable, BindableBool>();
-
-        /// <summary>
-        /// Gets or creates a <see cref="BindableBool"/> representing whether <paramref name="drawable"/> is using the closest <see cref="Drawable.Anchor">anchor point</see> within its
-        /// <see cref="Drawable.Parent">parent</see>.
-        /// </summary>
-        /// <returns>A <see cref="BindableBool"/> whose <see cref="Bindable{T}.Value"/> is <see langword="true"/> if the <see cref="IDrawable"/> is using the closest anchor point,
-        /// otherwise <see langword="false"/>.</returns>
-        public static BindableBool UsingClosestAnchor(this IDrawable drawable) =>
-            is_drawable_using_closest_anchor_lookup.GetValue(drawable, _ => new BindableBool(true));
     }
 }
diff --git a/osu.Game/Screens/Play/HUD/SkinnableInfo.cs b/osu.Game/Screens/Play/HUD/SkinnableInfo.cs
index 5c733a9bc1..a76634a1f0 100644
--- a/osu.Game/Screens/Play/HUD/SkinnableInfo.cs
+++ b/osu.Game/Screens/Play/HUD/SkinnableInfo.cs
@@ -55,7 +55,10 @@ namespace osu.Game.Screens.Play.HUD
             Scale = component.Scale;
             Anchor = component.Anchor;
             Origin = component.Origin;
-            UsingClosestAnchor = component.UsingClosestAnchor().Value;
+
+            UsingClosestAnchor =
+                // true if it's not an ISkinnableDrawable
+                !(component is ISkinnableDrawable skinnable) || skinnable.UsingClosestAnchor;
 
             if (component is Container<Drawable> container)
             {
diff --git a/osu.Game/Skinning/Editor/SkinSelectionHandler.cs b/osu.Game/Skinning/Editor/SkinSelectionHandler.cs
index 5acc949182..f4476b9de7 100644
--- a/osu.Game/Skinning/Editor/SkinSelectionHandler.cs
+++ b/osu.Game/Skinning/Editor/SkinSelectionHandler.cs
@@ -241,7 +241,7 @@ namespace osu.Game.Skinning.Editor
 
         private void updateDrawableAnchorIfUsingClosest(Drawable drawable)
         {
-            if (!drawable.UsingClosestAnchor().Value) return;
+            if (!(drawable is ISkinnableDrawable { UsingClosestAnchor: true })) return;
 
             var closestAnchor = getClosestAnchorForDrawable(drawable);
 
@@ -265,8 +265,8 @@ namespace osu.Game.Skinning.Editor
 
         protected override IEnumerable<MenuItem> GetContextMenuItemsForSelection(IEnumerable<SelectionBlueprint<ISkinnableDrawable>> selection)
         {
-            int checkAnchor(Drawable drawable) =>
-                drawable.UsingClosestAnchor().Value
+            static int checkAnchor(Drawable drawable) =>
+                drawable is ISkinnableDrawable { UsingClosestAnchor: true }
                     ? closest_text_hash
                     : (int)drawable.Anchor;
 
@@ -331,8 +331,12 @@ namespace osu.Game.Skinning.Editor
             {
                 var drawable = (Drawable)item;
 
-                var anchor = mapHashToAnchorSettings(hash, drawable);
+                var (usingClosest, anchor) =
+                    hash == closest_text_hash
+                        ? (true, getClosestAnchorForDrawable(drawable))
+                        : (false, (Anchor)hash);
 
+                item.UsingClosestAnchor = usingClosest;
                 updateDrawableAnchor(drawable, anchor);
             }
         }
@@ -344,20 +348,6 @@ namespace osu.Game.Skinning.Editor
             drawable.Position -= drawable.AnchorPosition - previousAnchor;
         }
 
-        private Anchor mapHashToAnchorSettings(int hash, Drawable drawable)
-        {
-            var isUsingClosestAnchor = drawable.UsingClosestAnchor();
-
-            if (hash == closest_text_hash)
-            {
-                isUsingClosestAnchor.Value = true;
-                return getClosestAnchorForDrawable(drawable);
-            }
-
-            isUsingClosestAnchor.Value = false;
-            return (Anchor)hash;
-        }
-
         private static void adjustScaleFromAnchor(ref Vector2 scale, Anchor reference)
         {
             // cancel out scale in axes we don't care about (based on which drag handle was used).