Merge pull request #18388 from peppy/editor-timing-waveforms

Add waveform comparison view to editor timing screen

osu.Android.props

@@ -52,7 +52,7 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="ppy.osu.Game.Resources" Version="2022.513.0" />
-    <PackageReference Include="ppy.osu.Framework.Android" Version="2022.521.0" />
+    <PackageReference Include="ppy.osu.Framework.Android" Version="2022.525.0" />
   </ItemGroup>
   <ItemGroup Label="Transitive Dependencies">
     <!-- Realm needs to be directly referenced in all Xamarin projects, as it will not pull in its transitive dependencies otherwise. -->

osu.Game.Tests/Visual/Editing/TestSceneTapTimingControl.cs

@@ -4,15 +4,15 @@
 using System.Linq;
 using NUnit.Framework;
 using osu.Framework.Allocation;
+using osu.Framework.Audio;
 using osu.Framework.Bindables;
 using osu.Framework.Graphics;
 using osu.Framework.Graphics.Containers;
 using osu.Framework.Testing;
 using osu.Game.Beatmaps.ControlPoints;
+using osu.Game.Graphics.Sprites;
 using osu.Game.Graphics.UserInterfaceV2;
 using osu.Game.Overlays;
-using osu.Game.Rulesets.Edit;
-using osu.Game.Rulesets.Osu;
 using osu.Game.Screens.Edit;
 using osu.Game.Screens.Edit.Timing;
 using osuTK;
@@ -22,9 +22,9 @@ namespace osu.Game.Tests.Visual.Editing
     [TestFixture]
     public class TestSceneTapTimingControl : EditorClockTestScene
     {
-        [Cached(typeof(EditorBeatmap))]
+        private EditorBeatmap editorBeatmap => editorBeatmapContainer?.EditorBeatmap;
-        [Cached(typeof(IBeatSnapProvider))]
+
-        private readonly EditorBeatmap editorBeatmap;
+        private TestSceneHitObjectComposer.EditorBeatmapContainer editorBeatmapContainer;
 
         [Cached]
         private readonly OverlayColourProvider colourProvider = new OverlayColourProvider(OverlayColourScheme.Blue);
@@ -33,38 +33,48 @@ namespace osu.Game.Tests.Visual.Editing
         private Bindable<ControlPointGroup> selectedGroup = new Bindable<ControlPointGroup>();
 
         private TapTimingControl control;
+        private OsuSpriteText timingInfo;
 
-        public TestSceneTapTimingControl()
+        [Resolved]
+        private AudioManager audio { get; set; }
+
+        [SetUpSteps]
+        public void SetUpSteps()
         {
-            var playableBeatmap = CreateBeatmap(new OsuRuleset().RulesetInfo);
+            AddStep("create beatmap", () =>
+            {
+                Beatmap.Value = new WaveformTestBeatmap(audio);
+            });
 
-            // Ensure time doesn't end while testing
+            AddStep("Create component", () =>
-            playableBeatmap.BeatmapInfo.Length = 1200000;
+            {
+                Child = editorBeatmapContainer = new TestSceneHitObjectComposer.EditorBeatmapContainer(Beatmap.Value)
+                {
+                    Children = new Drawable[]
+                    {
+                        new Container
+                        {
+                            Anchor = Anchor.Centre,
+                            Origin = Anchor.Centre,
+                            AutoSizeAxes = Axes.Y,
+                            Width = 400,
+                            Scale = new Vector2(1.5f),
+                            Child = control = new TapTimingControl(),
+                        },
+                        timingInfo = new OsuSpriteText(),
+                    }
+                };
 
-            editorBeatmap = new EditorBeatmap(playableBeatmap);
+                selectedGroup.Value = editorBeatmap.ControlPointInfo.Groups.First();
-
+            });
-            selectedGroup.Value = editorBeatmap.ControlPointInfo.Groups.First();
         }
 
-        protected override void LoadComplete()
+        protected override void Update()
         {
-            base.LoadComplete();
+            base.Update();
 
-            Beatmap.Value = CreateWorkingBeatmap(editorBeatmap.PlayableBeatmap);
+            if (selectedGroup.Value != null)
-            Beatmap.Disabled = true;
+                timingInfo.Text = $"offset: {selectedGroup.Value.Time:N2} bpm: {selectedGroup.Value.ControlPoints.OfType<TimingControlPoint>().First().BPM:N2}";
-
-            Children = new Drawable[]
-            {
-                new Container
-                {
-                    Anchor = Anchor.Centre,
-                    Origin = Anchor.Centre,
-                    AutoSizeAxes = Axes.Y,
-                    Width = 400,
-                    Scale = new Vector2(1.5f),
-                    Child = control = new TapTimingControl(),
-                }
-            };
         }
 
         [Test]
@@ -104,7 +114,13 @@ namespace osu.Game.Tests.Visual.Editing
                        .TriggerClick();
             });
 
-            AddSliderStep("BPM", 30, 400, 60, bpm => editorBeatmap.ControlPointInfo.TimingPoints.First().BeatLength = 60000f / bpm);
+            AddSliderStep("BPM", 30, 400, 128, bpm =>
+            {
+                if (editorBeatmap == null)
+                    return;
+
+                editorBeatmap.ControlPointInfo.TimingPoints.First().BeatLength = 60000f / bpm;
+            });
         }
 
         protected override void Dispose(bool isDisposing)

osu.Game/Beatmaps/ControlPoints/TimingControlPoint.cs

@@ -1,6 +1,8 @@
 // Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
 // See the LICENCE file in the repository root for full licence text.
 
+#nullable enable
+
 using osu.Framework.Bindables;
 using osu.Game.Beatmaps.Timing;
 using osu.Game.Graphics;

osu.Game/Screens/Edit/Timing/TapTimingControl.cs

@@ -24,8 +24,8 @@ namespace osu.Game.Screens.Edit.Timing
         [BackgroundDependencyLoader]
         private void load(OverlayColourProvider colourProvider, OsuColour colours)
         {
-            Height = 200;
             RelativeSizeAxes = Axes.X;
+            AutoSizeAxes = Axes.Y;
 
             CornerRadius = LabelledDrawable<Drawable>.CORNER_RADIUS;
             Masking = true;
@@ -39,20 +39,44 @@ namespace osu.Game.Screens.Edit.Timing
                 },
                 new GridContainer
                 {
-                    RelativeSizeAxes = Axes.Both,
+                    RelativeSizeAxes = Axes.X,
+                    AutoSizeAxes = Axes.Y,
                     RowDimensions = new[]
                     {
-                        new Dimension(),
+                        new Dimension(GridSizeMode.Absolute, 200),
                         new Dimension(GridSizeMode.Absolute, 60),
                     },
                     Content = new[]
                     {
                         new Drawable[]
                         {
-                            new MetronomeDisplay
+                            new Container
                             {
-                                Anchor = Anchor.Centre,
+                                RelativeSizeAxes = Axes.Both,
-                                Origin = Anchor.Centre,
+                                Children = new Drawable[]
+                                {
+                                    new GridContainer
+                                    {
+                                        RelativeSizeAxes = Axes.Both,
+                                        ColumnDimensions = new[]
+                                        {
+                                            new Dimension(GridSizeMode.AutoSize),
+                                            new Dimension()
+                                        },
+                                        Content = new[]
+                                        {
+                                            new Drawable[]
+                                            {
+                                                new MetronomeDisplay
+                                                {
+                                                    Anchor = Anchor.CentreLeft,
+                                                    Origin = Anchor.CentreLeft,
+                                                },
+                                                new WaveformComparisonDisplay(),
+                                            }
+                                        },
+                                    }
+                                }
                             }
                         },
                         new Drawable[]

osu.Game/Screens/Edit/Timing/WaveformComparisonDisplay.cs

@@ -0,0 +1,218 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+#nullable enable
+
+using System;
+using System.Linq;
+using osu.Framework.Allocation;
+using osu.Framework.Bindables;
+using osu.Framework.Graphics;
+using osu.Framework.Graphics.Audio;
+using osu.Framework.Graphics.Containers;
+using osu.Framework.Graphics.Shapes;
+using osu.Framework.Input.Events;
+using osu.Game.Beatmaps;
+using osu.Game.Beatmaps.ControlPoints;
+using osu.Game.Graphics.Sprites;
+using osu.Game.Graphics.UserInterfaceV2;
+using osu.Game.Overlays;
+using osuTK;
+using osuTK.Graphics;
+
+namespace osu.Game.Screens.Edit.Timing
+{
+    internal class WaveformComparisonDisplay : CompositeDrawable
+    {
+        private const int total_waveforms = 8;
+
+        private readonly BindableNumber<double> beatLength = new BindableDouble();
+
+        [Resolved]
+        private IBindable<WorkingBeatmap> beatmap { get; set; } = null!;
+
+        [Resolved]
+        private EditorBeatmap editorBeatmap { get; set; } = null!;
+
+        [Resolved]
+        private Bindable<ControlPointGroup?> selectedGroup { get; set; } = null!;
+
+        [Resolved]
+        private EditorClock editorClock { get; set; } = null!;
+
+        private TimingControlPoint timingPoint = TimingControlPoint.DEFAULT;
+
+        private int lastDisplayedBeatIndex;
+
+        private double selectedGroupStartTime;
+        private double selectedGroupEndTime;
+
+        private readonly IBindableList<ControlPointGroup> controlPointGroups = new BindableList<ControlPointGroup>();
+
+        public WaveformComparisonDisplay()
+        {
+            RelativeSizeAxes = Axes.Both;
+
+            CornerRadius = LabelledDrawable<Drawable>.CORNER_RADIUS;
+            Masking = true;
+        }
+
+        protected override void LoadComplete()
+        {
+            base.LoadComplete();
+
+            for (int i = 0; i < total_waveforms; i++)
+            {
+                AddInternal(new WaveformRow
+                {
+                    RelativeSizeAxes = Axes.Both,
+                    RelativePositionAxes = Axes.Both,
+                    Height = 1f / total_waveforms,
+                    Y = (float)i / total_waveforms,
+                });
+            }
+
+            AddInternal(new Circle
+            {
+                Anchor = Anchor.Centre,
+                Origin = Anchor.Centre,
+                Colour = Color4.White,
+                RelativeSizeAxes = Axes.Y,
+                Width = 3,
+            });
+
+            selectedGroup.BindValueChanged(_ => updateTimingGroup(), true);
+
+            controlPointGroups.BindTo(editorBeatmap.ControlPointInfo.Groups);
+            controlPointGroups.BindCollectionChanged((_, __) => updateTimingGroup());
+
+            beatLength.BindValueChanged(_ => showFrom(lastDisplayedBeatIndex), true);
+        }
+
+        private void updateTimingGroup()
+        {
+            beatLength.UnbindBindings();
+
+            selectedGroupStartTime = 0;
+            selectedGroupEndTime = beatmap.Value.Track.Length;
+
+            var tcp = selectedGroup.Value?.ControlPoints.OfType<TimingControlPoint>().FirstOrDefault();
+
+            if (tcp == null)
+            {
+                timingPoint = new TimingControlPoint();
+                return;
+            }
+
+            timingPoint = tcp;
+            beatLength.BindTo(timingPoint.BeatLengthBindable);
+
+            selectedGroupStartTime = selectedGroup.Value?.Time ?? 0;
+
+            var nextGroup = editorBeatmap.ControlPointInfo.TimingPoints
+                                         .SkipWhile(g => g != tcp)
+                                         .Skip(1)
+                                         .FirstOrDefault();
+
+            if (nextGroup != null)
+                selectedGroupEndTime = nextGroup.Time;
+        }
+
+        protected override bool OnHover(HoverEvent e) => true;
+
+        protected override bool OnMouseMove(MouseMoveEvent e)
+        {
+            float trackLength = (float)beatmap.Value.Track.Length;
+            int totalBeatsAvailable = (int)(trackLength / timingPoint.BeatLength);
+
+            Scheduler.AddOnce(showFrom, (int)(e.MousePosition.X / DrawWidth * totalBeatsAvailable));
+
+            return base.OnMouseMove(e);
+        }
+
+        protected override void Update()
+        {
+            base.Update();
+
+            if (!IsHovered)
+            {
+                int currentBeat = (int)Math.Floor((editorClock.CurrentTimeAccurate - selectedGroupStartTime) / timingPoint.BeatLength);
+
+                showFrom(currentBeat);
+            }
+        }
+
+        private void showFrom(int beatIndex)
+        {
+            if (lastDisplayedBeatIndex == beatIndex)
+                return;
+
+            // Chosen as a pretty usable number across all BPMs.
+            // Optimally we'd want this to scale with the BPM in question, but performing
+            // scaling of the display is both expensive in resampling, and decreases usability
+            // (as it is harder to track the waveform when making realtime adjustments).
+            const float visible_width = 300;
+
+            float trackLength = (float)beatmap.Value.Track.Length;
+            float scale = trackLength / visible_width;
+
+            // Start displaying from before the current beat
+            beatIndex -= total_waveforms / 2;
+
+            foreach (var row in InternalChildren.OfType<WaveformRow>())
+            {
+                // offset to the required beat index.
+                double time = selectedGroupStartTime + beatIndex * timingPoint.BeatLength;
+
+                float offset = (float)(time - visible_width / 2) / trackLength * scale;
+
+                row.Alpha = time < selectedGroupStartTime || time > selectedGroupEndTime ? 0.2f : 1;
+                row.WaveformOffset = -offset;
+                row.WaveformScale = new Vector2(scale, 1);
+                row.BeatIndex = beatIndex++;
+            }
+
+            lastDisplayedBeatIndex = beatIndex;
+        }
+
+        internal class WaveformRow : CompositeDrawable
+        {
+            private OsuSpriteText beatIndexText = null!;
+            private WaveformGraph waveformGraph = null!;
+
+            [Resolved]
+            private OverlayColourProvider colourProvider { get; set; } = null!;
+
+            [BackgroundDependencyLoader]
+            private void load(IBindable<WorkingBeatmap> beatmap)
+            {
+                InternalChildren = new Drawable[]
+                {
+                    waveformGraph = new WaveformGraph
+                    {
+                        RelativeSizeAxes = Axes.Both,
+                        RelativePositionAxes = Axes.Both,
+                        Waveform = beatmap.Value.Waveform,
+                        Resolution = 1,
+
+                        BaseColour = colourProvider.Colour0,
+                        LowColour = colourProvider.Colour1,
+                        MidColour = colourProvider.Colour2,
+                        HighColour = colourProvider.Colour4,
+                    },
+                    beatIndexText = new OsuSpriteText
+                    {
+                        Anchor = Anchor.CentreLeft,
+                        Origin = Anchor.CentreLeft,
+                        Padding = new MarginPadding(5),
+                        Colour = colourProvider.Content2
+                    }
+                };
+            }
+
+            public int BeatIndex { set => beatIndexText.Text = value.ToString(); }
+            public Vector2 WaveformScale { set => waveformGraph.Scale = value; }
+            public float WaveformOffset { set => waveformGraph.X = value; }
+        }
+    }
+}

osu.Game/osu.Game.csproj

@@ -35,7 +35,7 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="Realm" Version="10.11.2" />
-    <PackageReference Include="ppy.osu.Framework" Version="2022.521.0" />
+    <PackageReference Include="ppy.osu.Framework" Version="2022.525.0" />
     <PackageReference Include="ppy.osu.Game.Resources" Version="2022.513.0" />
     <PackageReference Include="Sentry" Version="3.17.1" />
     <PackageReference Include="SharpCompress" Version="0.31.0" />

osu.iOS.props

@@ -61,7 +61,7 @@
     <Reference Include="System.Net.Http" />
   </ItemGroup>
   <ItemGroup Label="Package References">
-    <PackageReference Include="ppy.osu.Framework.iOS" Version="2022.521.0" />
+    <PackageReference Include="ppy.osu.Framework.iOS" Version="2022.525.0" />
     <PackageReference Include="ppy.osu.Game.Resources" Version="2022.513.0" />
   </ItemGroup>
   <!-- See https://github.com/dotnet/runtime/issues/35988 (can be removed after Xamarin uses net6.0) -->
@@ -84,7 +84,7 @@
     <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="5.0.14" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite.Core" Version="5.0.14" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
-    <PackageReference Include="ppy.osu.Framework" Version="2022.521.0" />
+    <PackageReference Include="ppy.osu.Framework" Version="2022.525.0" />
     <PackageReference Include="SharpCompress" Version="0.31.0" />
     <PackageReference Include="NUnit" Version="3.13.3" />
     <PackageReference Include="System.ComponentModel.Annotations" Version="5.0.0" />