wip

Elementary/Audio/AudioManager.cs

@@ -5,7 +5,6 @@ using Discord;
 using Discord.Audio;
 using Discord.WebSocket;
 using Elementary.Dictionary;
-using ManagedBass;
 using Microsoft.Extensions.DependencyInjection;
 using NAudio.Wave;
 using NLog;
@@ -21,12 +20,14 @@ public class AudioManager
 
     private SozaiAPI _sozaiAPI;
     private VoicevoxAPI _voicevoxAPI;
+    private Ytdlp _ytdlp;
 
     private AudioConverter _audioConverter;
 
     private PlaybackQueue _playbackQueue;
     private EmojiDictionary _emojiDictionary;
     private DictionaryDB _dictionaryDB;
+    private Mecab _mecab;
     // private AudioMixer _audioMixer;
 
     private ILogger _logger;
@@ -34,15 +35,17 @@ public class AudioManager
     public bool isConnected;
 
     public AudioManager(IServiceProvider services, DiscordSocketClient client, SozaiAPI sozaiApi,
-        VoicevoxAPI voicevoxApi, EmojiDictionary emojiDictionary, DictionaryDB dictionaryDB)
+        VoicevoxAPI voicevoxApi, EmojiDictionary emojiDictionary, DictionaryDB dictionaryDB, Mecab mecab, Ytdlp ytdlp)
     {
         _services = services;
         _client = client;
         _sozaiAPI = sozaiApi;
         _voicevoxAPI = voicevoxApi;
+        _ytdlp = ytdlp;
         _audioConverter = new();
         _emojiDictionary = emojiDictionary;
         _dictionaryDB = dictionaryDB;
+        _mecab = mecab;
         _logger = LogManager.GetCurrentClassLogger();
     }
 
@@ -88,6 +91,28 @@ public class AudioManager
         // _audioMixer.AddStream(wave);
         await wave.CopyToAsync(_audioStream);
         // GC.Collect();
+
+        // try
+        // {
+        //     await wave.CopyToAsync(_audioStream);
+        // }
+        // catch (Exception e)
+        // {
+        //     _logger.Log(LogLevel.Error, e);
+        // }
+        // finally
+        // {
+        //     await _audioStream.DisposeAsync();
+        // }
+    }
+    
+    public async Task PlayYoutube(string url)
+    {
+        var stream = await _ytdlp.GetStream(url);
+        
+        await using var wave = _audioConverter.CreateStreamFromStream(stream, 0.1f);
+        // _audioMixer.AddStream(wave);
+        await wave.CopyToAsync(_audioStream);
     }
 
     public async Task PlayText(string text)
@@ -98,25 +123,42 @@ public class AudioManager
         
         text = Regex.Replace(text, @"<:[\w]+:[\d]+>", m => m.Value.Split(":")[1]); // <:emoji:123456789> -> emoji
         text = Regex.Replace(text, @"<a:[\w]+:[\d]+>", m => m.Value.Split(":")[1]); // <:emoji:123456789> -> emoji
-
-        text = _dictionaryDB.Replace(text);
         
-        text = _emojiDictionary.Replace(text);
+        text = text.Replace("～", "ー");
 
         float volume = 0.12f;
 
         Stream? stream = await _sozaiAPI.GetAudioStream(text);
         if (stream == null)
         {
+            text = _dictionaryDB.Replace(text);
+        
+            text = _emojiDictionary.Replace(text);
+        
+            text = await _mecab.ParseToKana(text);
+            
             stream = await _voicevoxAPI.Speak(text);
             volume = 0.8f;
             if (stream == null) return;
         }
-
+        
         await using var wave = _audioConverter.CreateStreamFromStream(stream, volume);
         // _audioMixer.AddStream(wave);
         await wave.CopyToAsync(_audioStream);
         // GC.Collect();
+        
+        // try
+        // {
+        //     await wave.CopyToAsync(_audioStream);
+        // }
+        // catch (Exception e)
+        // {
+        //     _logger.Log(LogLevel.Error, e);
+        // }
+        // finally
+        // {
+        //     await _audioStream.DisposeAsync();
+        // }
     }
 
     public async Task StopAudio()

Elementary/Audio/AudioMixer.cs

@@ -1,80 +1,80 @@
-using ManagedBass;
+// using ManagedBass;
-using ManagedBass.Mix;
+// using ManagedBass.Mix;
-
+//
-namespace Elementary.Audio;
+// namespace Elementary.Audio;
-
+//
-public class AudioMixer
+// public class AudioMixer
-{
+// {
-    private readonly int _mixerStream;
+//     private readonly int _mixerStream;
-    private MemoryStream?[] _inputStreams;
+//     private MemoryStream?[] _inputStreams;
-    private Stream _outStream;
+//     private Stream _outStream;
-    private object _lock = new();
+//     private object _lock = new();
-
+//
-    public AudioMixer(Stream outStream)
+//     public AudioMixer(Stream outStream)
-    {
+//     {
-        _outStream = outStream;
+//         _outStream = outStream;
-        Bass.Init(-1, 44100, DeviceInitFlags.NoSpeakerAssignment, IntPtr.Zero);
+//         Bass.Init(-1, 44100, DeviceInitFlags.NoSpeakerAssignment, IntPtr.Zero);
-        _mixerStream = BassMix.CreateMixerStream(44100, 2, BassFlags.Float | BassFlags.MixerNonStop);
+//         _mixerStream = BassMix.CreateMixerStream(44100, 2, BassFlags.Float | BassFlags.MixerNonStop);
-        _inputStreams = new MemoryStream[10];
+//         _inputStreams = new MemoryStream[10];
-
+//
-        Bass.ChannelPlay(_mixerStream);
+//         Bass.ChannelPlay(_mixerStream);
-    }
+//     }
-
+//
-    /// <summary>
+//     /// <summary>
-    /// Add a stream to the mixer.
+//     /// Add a stream to the mixer.
-    /// wait for the stream to finish playing 
+//     /// wait for the stream to finish playing 
-    /// </summary>
+//     /// </summary>
-    /// <param name="stream"></param>
+//     /// <param name="stream"></param>
-    /// <param name="volume"></param>
+//     /// <param name="volume"></param>
-    public void AddStream(Stream stream, float volume = 1.0f)
+//     public void AddStream(Stream stream, float volume = 1.0f)
-    {
+//     {
-        Console.WriteLine("Adding stream to mixer");
+//         Console.WriteLine("Adding stream to mixer");
-        byte[] pcmBytes;
+//         byte[] pcmBytes;
-        using (MemoryStream ms = new())
+//         using (MemoryStream ms = new())
-        {
+//         {
-            stream.CopyTo(ms);
+//             stream.CopyTo(ms);
-            pcmBytes = ms.ToArray();
+//             pcmBytes = ms.ToArray();
-        }
+//         }
-
+//
-        lock (_lock)
+//         lock (_lock)
-        {
+//         {
-            for (var i = 0; i < _inputStreams.Length; i++)
+//             for (var i = 0; i < _inputStreams.Length; i++)
-            {
+//             {
-                if (_inputStreams[i] == null)
+//                 if (_inputStreams[i] == null)
-                {
+//                 {
-                    _inputStreams[i] = new MemoryStream(pcmBytes);
+//                     _inputStreams[i] = new MemoryStream(pcmBytes);
-                    break;
+//                     break;
-                }
+//                 }
-            }
+//             }
-        }
+//         }
-        
+//         
-        var channel = Bass.CreateStream(pcmBytes, 0, pcmBytes.Length, BassFlags.Float);
+//         var channel = Bass.CreateStream(pcmBytes, 0, pcmBytes.Length, BassFlags.Float);
-        Bass.ChannelSetAttribute(channel, ChannelAttribute.Volume, volume);
+//         Bass.ChannelSetAttribute(channel, ChannelAttribute.Volume, volume);
-        BassMix.MixerAddChannel(_mixerStream, channel, BassFlags.Default);
+//         BassMix.MixerAddChannel(_mixerStream, channel, BassFlags.Default);
-        
+//         
-        int length = Bass.ChannelGetData(_mixerStream, new byte[4096], 4096);
+//         int length = Bass.ChannelGetData(_mixerStream, new byte[4096], 4096);
-        byte[] buffer = new byte[length];
+//         byte[] buffer = new byte[length];
-        length = Bass.ChannelGetData(_mixerStream, buffer, length);
+//         length = Bass.ChannelGetData(_mixerStream, buffer, length);
-        _outStream.Write(buffer, 0, length);
+//         _outStream.Write(buffer, 0, length);
-    }
+//     }
-
+//
-    public void Stop()
+//     public void Stop()
-    {
+//     {
-        Bass.ChannelStop(_mixerStream);
+//         Bass.ChannelStop(_mixerStream);
-    }
+//     }
-
+//
-    public void Dispose()
+//     public void Dispose()
-    {
+//     {
-        lock (_lock)
+//         lock (_lock)
-        {
+//         {
-            foreach (var stream in _inputStreams)
+//             foreach (var stream in _inputStreams)
-            {
+//             {
-                stream?.Dispose();
+//                 stream?.Dispose();
-            }
+//             }
-        }
+//         }
-        
+//         
-        _outStream.Flush();
+//         _outStream.Flush();
-        Bass.StreamFree(_mixerStream);
+//         Bass.StreamFree(_mixerStream);
-        Bass.Free();
+//         Bass.Free();
-    }
+//     }
-}
+// }

Elementary/Audio/MessageHandler.cs

@@ -7,19 +7,24 @@ public class MessageHandler
 {
     // private IAudioClient _audioClient;
     private PlaybackQueue _playbackQueue;
-    
+
     public MessageHandler(PlaybackQueue playbackQueue)
     {
         _playbackQueue = playbackQueue;
     }
-    
+
-    
+
     public async Task HandleMessage(SocketMessage message)
     {
-        await _playbackQueue.Enqueue(new PlaybackJob()
+        List<String> lines = message.Content.Split("\n").ToList();
+
+        foreach (var line in lines)
         {
-            Type = JobType.Text,
+            await _playbackQueue.Enqueue(new PlaybackJob()
-            Text = message.Content
+            {
-        });
+                Type = JobType.Text,
+                Text = line
+            });
+        }
     }
 }

Elementary/Audio/SozaiAPI.cs

@@ -1,4 +1,5 @@
 using System.Text.Json;
+using Microsoft.Extensions.Caching.Memory;
 using NLog;
 
 namespace Elementary.Audio;
@@ -9,6 +10,8 @@ public class SozaiAPI
     
     private Asset[] Assets;
 
+    private MemoryCache _cache;
+
     private class Asset
     {
         /// <summary>
@@ -29,6 +32,7 @@ public class SozaiAPI
     {
         _client = new HttpClient();
         _logger = LogManager.GetCurrentClassLogger();
+        _cache = new MemoryCache(new MemoryCacheOptions());
     }
 
     public async Task Setup(string url)
@@ -47,10 +51,17 @@ public class SozaiAPI
         if (asset == null) return null;
 
         _logger.Info($"Requested {asset.names[0]}");
+        
+        if (_cache.TryGetValue(asset.url, out Stream? stream))
+        {
+            _logger.Info($"Cache hit {asset.url}");
+            return stream;
+        }
 
         var response = await _client.GetAsync(asset.url);
         _logger.Info($"Got response {response.StatusCode}");
-        var stream = await response.Content.ReadAsStreamAsync();
+        stream = await response.Content.ReadAsStreamAsync();
+        _cache.Set(asset.url, stream);
         return stream;
     }
 }

Elementary/Audio/VoicevoxAPI.cs

@@ -1,5 +1,6 @@
 using System.Net.Http.Json;
 using System.Text;
+using Microsoft.Extensions.Caching.Memory;
 using NLog;
 
 namespace Elementary.Audio;
@@ -9,6 +10,7 @@ public class VoicevoxAPI
     private UriBuilder _APIRootUrl;
     private HttpClient _client;
     private ILogger _logger;
+    private MemoryCache _cache;
 
     public async Task Setup(string url)
     {
@@ -17,6 +19,7 @@ public class VoicevoxAPI
         _APIRootUrl = new UriBuilder($"{_url.Scheme}://{_url.Host}:{_url.Port}");
         _client = new HttpClient();
         _logger = LogManager.GetCurrentClassLogger();
+        _cache = new MemoryCache(new MemoryCacheOptions());
     }
 
     /// <summary>
@@ -28,11 +31,18 @@ public class VoicevoxAPI
     public async Task<Stream?> Speak(string text, string speaker = "47")
     {
         _logger.Info($"Requested TTS {text}");
+        
+        if (_cache.TryGetValue(text, out Stream? stream))
+        {
+            _logger.Info($"Cache hit {text}");
+            return stream;
+        }
 
         var query = await GetAudioQuery(text, speaker);
         if (query == null) return null;
-        var stream = await GetAudioStream(query, speaker);
+        stream = await GetAudioStream(query, speaker);
         if (stream == null) return null;
+        _cache.Set(text, stream);
         return stream;
     }
 

Elementary/Audio/Ytdlp.cs

@@ -0,0 +1,39 @@
+using System.Diagnostics;
+using System.Text;
+using NLog;
+
+namespace Elementary.Audio;
+
+public class Ytdlp
+{
+    private readonly ILogger _logger;
+
+    public Ytdlp()
+    {
+        _logger = LogManager.GetCurrentClassLogger();
+    }
+
+    public async Task<Stream> GetStream(string url)
+    {
+        var process = new Process
+        {
+            StartInfo =
+            {
+                FileName = "yt-dlp",
+                Arguments = $"-f bestaudio -o - {url}",
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                UseShellExecute = false,
+                CreateNoWindow = true
+            },
+            EnableRaisingEvents = true
+        };
+
+        var stream = new MemoryStream();
+        process.Start();
+        process.StandardOutput.BaseStream.CopyTo(stream);
+        process.WaitForExit();
+        stream.Position = 0;
+        return stream;
+    }
+}

Elementary/Commands/MessageCommands.cs

@@ -112,4 +112,11 @@ public class MessageCommands : ModuleBase<SocketCommandContext>
         result += "```";
         return ReplyAsync(result);
     }
+    
+    [Command("ytdlp", RunMode = RunMode.Async)]
+    [Summary("Play sound from Youtube URL.")]
+    public async Task YtdlpAsync([Summary("Youtube URL")] string url)
+    {
+        await _audioManager.PlayYoutube(url);
+    }
 }

Elementary/Config/Configuration.cs

@@ -10,6 +10,13 @@ public class AppSettings
     public SozaiSettings SozaiSettings { get; set; }
     public EmojiSettings EmojiSettings { get; set; }
     public VoicevoxSettings VoicevoxSettings { get; set; }
+    public MecabSettings MecabSettings { get; set; }
+}
+
+public class MecabSettings
+{
+    public bool enabled { get; set; }
+    public string? dictionaryPath { get; set; }
 }
 
 public class VoicevoxSettings

Elementary/Dictionary/Mecab.cs

@@ -0,0 +1,67 @@
+using System.Diagnostics;
+using System.Text;
+using System.Text.RegularExpressions;
+
+namespace Elementary.Dictionary;
+
+public class Mecab
+{
+    private string? _dictionaryPath;
+    private bool _enabled;
+
+    public async Task Setup(bool enabled, string? dictionaryPath)
+    {
+        _dictionaryPath = dictionaryPath;
+        _enabled = enabled;
+        await Task.CompletedTask;
+    }
+
+    public async Task<String> ParseToKana(string text)
+    {
+        if (!_enabled)
+        {
+            return text;
+        }
+
+        // "Apple PencilあああiPad" -> "Apple Pencil iPad"
+        var englishWords = Regex.Matches(text, @"[a-zA-Z]+").Select(m => m.Value).ToList();
+        var englishWordsString = string.Join(" ", englishWords);
+
+        String args = $"-Oyomi";
+        if (_dictionaryPath != null)
+        {
+            args += $" -d {_dictionaryPath}";
+        }
+
+        var process = new Process
+        {
+            StartInfo =
+            {
+                FileName = "mecab",
+                Arguments = args,
+                RedirectStandardInput = true,
+                RedirectStandardOutput = true,
+                StandardOutputEncoding = Encoding.UTF8,
+                StandardInputEncoding = Encoding.UTF8,
+                RedirectStandardError = true,
+                UseShellExecute = false,
+                CreateNoWindow = true
+            }
+        };
+        process.Start();
+        await process.StandardInput.WriteLineAsync(englishWordsString);
+        await process.StandardInput.FlushAsync();
+        process.StandardInput.Close();
+        var result = await process.StandardOutput.ReadToEndAsync();
+        process.WaitForExit();
+
+        // replace english words with kana
+        var kanaWords = result.Split(" ").Select(m => m.Trim()).ToList();
+        for (var i = 0; i < englishWords.Count; i++)
+        {
+            text = text.Replace(englishWords[i], kanaWords[i]);
+        }
+
+        return text;
+    }
+}

Elementary/Elementary.csproj

@@ -17,12 +17,10 @@
     </ItemGroup>
 
     <ItemGroup>
-        <PackageReference Include="CSCore" Version="1.2.1.2"/>
         <PackageReference Include="Discord.Net" Version="3.11.0"/>
         <PackageReference Include="Discord.Net.Core" Version="3.11.0"/>
         <PackageReference Include="LiteDB" Version="5.0.17" />
-        <PackageReference Include="ManagedBass" Version="3.1.1"/>
+        <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="7.0.0" />
-        <PackageReference Include="ManagedBass.Mix" Version="3.1.1"/>
         <PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0"/>
         <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.4"/>
         <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="7.0.0"/>

Elementary/Program.cs

@@ -7,7 +7,6 @@ using Discord.WebSocket;
 using Elementary.Audio;
 using Elementary.Commands;
 using Elementary.Dictionary;
-using ManagedBass;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using NAudio.Wave;
@@ -30,6 +29,8 @@ public class Program
     private VoicevoxAPI _voicevoxAPI;
     private EmojiDictionary _emojiDictionary;
     private DictionaryDB _dictionaryDB;
+    private Mecab _mecab;
+    private Ytdlp _ytdlp;
 
     public static async Task Main(string[] args)
     {
@@ -73,15 +74,19 @@ public class Program
             .AddSingleton<MessageHandler>()
             .AddSingleton<EmojiDictionary>()
             .AddSingleton<DictionaryDB>()
+            .AddSingleton<Mecab>()
+            .AddSingleton<Ytdlp>()
             .BuildServiceProvider();
 
         _logger = LogManager.GetCurrentClassLogger();
 
         _sozaiAPI = _services.GetRequiredService<SozaiAPI>();
         _voicevoxAPI = _services.GetRequiredService<VoicevoxAPI>();
+        _ytdlp = _services.GetRequiredService<Ytdlp>();
 
         _emojiDictionary = _services.GetRequiredService<EmojiDictionary>();
         _dictionaryDB = _services.GetRequiredService<DictionaryDB>();
+        _mecab = _services.GetRequiredService<Mecab>();
 
         _handler = new(_client, _commands, _services, _services.GetRequiredService<AudioManager>(),
             _services.GetRequiredService<MessageHandler>());
@@ -96,6 +101,8 @@ public class Program
         await _voicevoxAPI.Setup(configuration.AppSettings.VoicevoxSettings.Url);
         await _emojiDictionary.Setup(configuration.AppSettings.EmojiSettings.DictionaryPath);
         await _dictionaryDB.Setup();
+        await _mecab.Setup(configuration.AppSettings.MecabSettings.enabled,
+            configuration.AppSettings.MecabSettings.dictionaryPath);
         await _client.LoginAsync(TokenType.Bot, configuration.AppSettings.DiscordSettings.Token);
         await _client.StartAsync();
         await _client.SetActivityAsync(new Game("!join"));

Elementary/appsettings.json

@@ -18,6 +18,9 @@
     },
     "EmojiSettings": {
       "DictionaryPath": "emoji-ja\\data\\emoji_ja.json"
+    },
+    "MecabSettings": {
+      "Enabled": true
     }
   }
 }