sim1222/Elementary
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
sim1222 2023-08-09 01:35:43 +09:00
parent 21bc3bd775
commit 7d60313cf7
12 changed files with 295 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
Elementary/Audio/AudioManager.cs
View File

@ -5,7 +5,6 @@ using Discord;
using Discord.Audio;
using Discord.WebSocket;
using Elementary.Dictionary;
using ManagedBass;
using Microsoft.Extensions.DependencyInjection;
using NAudio.Wave;
using NLog;
@ -21,12 +20,14 @@ public class AudioManager
private SozaiAPI _sozaiAPI;
private VoicevoxAPI _voicevoxAPI;
private Ytdlp _ytdlp;
private AudioConverter _audioConverter;
private PlaybackQueue _playbackQueue;
private EmojiDictionary _emojiDictionary;
private DictionaryDB _dictionaryDB;
private Mecab _mecab;
// private AudioMixer _audioMixer;
private ILogger _logger;
@ -34,15 +35,17 @@ public class AudioManager
public bool isConnected;
public AudioManager(IServiceProvider services, DiscordSocketClient client, SozaiAPI sozaiApi,
VoicevoxAPI voicevoxApi, EmojiDictionary emojiDictionary, DictionaryDB dictionaryDB)
VoicevoxAPI voicevoxApi, EmojiDictionary emojiDictionary, DictionaryDB dictionaryDB, Mecab mecab, Ytdlp ytdlp)
{
_services = services;
_client = client;
_sozaiAPI = sozaiApi;
_voicevoxAPI = voicevoxApi;
_ytdlp = ytdlp;
_audioConverter = new();
_emojiDictionary = emojiDictionary;
_dictionaryDB = dictionaryDB;
_mecab = mecab;
_logger = LogManager.GetCurrentClassLogger();
}
@ -88,6 +91,28 @@ public class AudioManager
// _audioMixer.AddStream(wave);
await wave.CopyToAsync(_audioStream);
// GC.Collect();
// try
// {
// await wave.CopyToAsync(_audioStream);
// }
// catch (Exception e)
// {
// _logger.Log(LogLevel.Error, e);
// }
// finally
// {
// await _audioStream.DisposeAsync();
// }
}
public async Task PlayYoutube(string url)
{
var stream = await _ytdlp.GetStream(url);
await using var wave = _audioConverter.CreateStreamFromStream(stream, 0.1f);
// _audioMixer.AddStream(wave);
await wave.CopyToAsync(_audioStream);
}
public async Task PlayText(string text)
@ -99,15 +124,19 @@ public class AudioManager
text = Regex.Replace(text, @"<:[\w]+:[\d]+>", m => m.Value.Split(":")[1]); // <:emoji:123456789> -> emoji
text = Regex.Replace(text, @"<a:[\w]+:[\d]+>", m => m.Value.Split(":")[1]); // <:emoji:123456789> -> emoji
text = _dictionaryDB.Replace(text);
text = _emojiDictionary.Replace(text);
text = text.Replace("～", "ー");
float volume = 0.12f;
Stream? stream = await _sozaiAPI.GetAudioStream(text);
if (stream == null)
{
text = _dictionaryDB.Replace(text);
text = _emojiDictionary.Replace(text);
text = await _mecab.ParseToKana(text);
stream = await _voicevoxAPI.Speak(text);
volume = 0.8f;
if (stream == null) return;
@ -117,6 +146,19 @@ public class AudioManager
// _audioMixer.AddStream(wave);
await wave.CopyToAsync(_audioStream);
// GC.Collect();
// try
// {
// await wave.CopyToAsync(_audioStream);
// }
// catch (Exception e)
// {
// _logger.Log(LogLevel.Error, e);
// }
// finally
// {
// await _audioStream.DisposeAsync();
// }
}
public async Task StopAudio()

160
Elementary/Audio/AudioMixer.cs
View File

@ -1,80 +1,80 @@
using ManagedBass;
using ManagedBass.Mix;
namespace Elementary.Audio;
public class AudioMixer
{
private readonly int _mixerStream;
private MemoryStream?[] _inputStreams;
private Stream _outStream;
private object _lock = new();
public AudioMixer(Stream outStream)
{
_outStream = outStream;
Bass.Init(-1, 44100, DeviceInitFlags.NoSpeakerAssignment, IntPtr.Zero);
_mixerStream = BassMix.CreateMixerStream(44100, 2, BassFlags.Float | BassFlags.MixerNonStop);
_inputStreams = new MemoryStream[10];
Bass.ChannelPlay(_mixerStream);
}
/// <summary>
/// Add a stream to the mixer.
/// wait for the stream to finish playing
/// </summary>
/// <param name="stream"></param>
/// <param name="volume"></param>
public void AddStream(Stream stream, float volume = 1.0f)
{
Console.WriteLine("Adding stream to mixer");
byte[] pcmBytes;
using (MemoryStream ms = new())
{
stream.CopyTo(ms);
pcmBytes = ms.ToArray();
}
lock (_lock)
{
for (var i = 0; i < _inputStreams.Length; i++)
{
if (_inputStreams[i] == null)
{
_inputStreams[i] = new MemoryStream(pcmBytes);
break;
}
}
}
var channel = Bass.CreateStream(pcmBytes, 0, pcmBytes.Length, BassFlags.Float);
Bass.ChannelSetAttribute(channel, ChannelAttribute.Volume, volume);
BassMix.MixerAddChannel(_mixerStream, channel, BassFlags.Default);
int length = Bass.ChannelGetData(_mixerStream, new byte[4096], 4096);
byte[] buffer = new byte[length];
length = Bass.ChannelGetData(_mixerStream, buffer, length);
_outStream.Write(buffer, 0, length);
}
public void Stop()
{
Bass.ChannelStop(_mixerStream);
}
public void Dispose()
{
lock (_lock)
{
foreach (var stream in _inputStreams)
{
stream?.Dispose();
}
}
_outStream.Flush();
Bass.StreamFree(_mixerStream);
Bass.Free();
}
}
// using ManagedBass;
// using ManagedBass.Mix;
//
// namespace Elementary.Audio;
//
// public class AudioMixer
// {
// private readonly int _mixerStream;
// private MemoryStream?[] _inputStreams;
// private Stream _outStream;
// private object _lock = new();
//
// public AudioMixer(Stream outStream)
// {
// _outStream = outStream;
// Bass.Init(-1, 44100, DeviceInitFlags.NoSpeakerAssignment, IntPtr.Zero);
// _mixerStream = BassMix.CreateMixerStream(44100, 2, BassFlags.Float | BassFlags.MixerNonStop);
// _inputStreams = new MemoryStream[10];
//
// Bass.ChannelPlay(_mixerStream);
// }
//
// /// <summary>
// /// Add a stream to the mixer.
// /// wait for the stream to finish playing
// /// </summary>
// /// <param name="stream"></param>
// /// <param name="volume"></param>
// public void AddStream(Stream stream, float volume = 1.0f)
// {
// Console.WriteLine("Adding stream to mixer");
// byte[] pcmBytes;
// using (MemoryStream ms = new())
// {
// stream.CopyTo(ms);
// pcmBytes = ms.ToArray();
// }
//
// lock (_lock)
// {
// for (var i = 0; i < _inputStreams.Length; i++)
// {
// if (_inputStreams[i] == null)
// {
// _inputStreams[i] = new MemoryStream(pcmBytes);
// break;
// }
// }
// }
//
// var channel = Bass.CreateStream(pcmBytes, 0, pcmBytes.Length, BassFlags.Float);
// Bass.ChannelSetAttribute(channel, ChannelAttribute.Volume, volume);
// BassMix.MixerAddChannel(_mixerStream, channel, BassFlags.Default);
//
// int length = Bass.ChannelGetData(_mixerStream, new byte[4096], 4096);
// byte[] buffer = new byte[length];
// length = Bass.ChannelGetData(_mixerStream, buffer, length);
// _outStream.Write(buffer, 0, length);
// }
//
// public void Stop()
// {
// Bass.ChannelStop(_mixerStream);
// }
//
// public void Dispose()
// {
// lock (_lock)
// {
// foreach (var stream in _inputStreams)
// {
// stream?.Dispose();
// }
// }
//
// _outStream.Flush();
// Bass.StreamFree(_mixerStream);
// Bass.Free();
// }
// }

7
Elementary/Audio/MessageHandler.cs
View File

@ -15,11 +15,16 @@ public class MessageHandler
public async Task HandleMessage(SocketMessage message)
{
List<String> lines = message.Content.Split("\n").ToList();
foreach (var line in lines)
{
await _playbackQueue.Enqueue(new PlaybackJob()
{
Type = JobType.Text,
Text = message.Content
Text = line
});
}
}
}

13
Elementary/Audio/SozaiAPI.cs
View File

@ -1,4 +1,5 @@
using System.Text.Json;
using Microsoft.Extensions.Caching.Memory;
using NLog;
namespace Elementary.Audio;
@ -9,6 +10,8 @@ public class SozaiAPI
private Asset[] Assets;
private MemoryCache _cache;
private class Asset
{
/// <summary>
@ -29,6 +32,7 @@ public class SozaiAPI
{
_client = new HttpClient();
_logger = LogManager.GetCurrentClassLogger();
_cache = new MemoryCache(new MemoryCacheOptions());
}
public async Task Setup(string url)
@ -48,9 +52,16 @@ public class SozaiAPI
_logger.Info($"Requested {asset.names[0]}");
if (_cache.TryGetValue(asset.url, out Stream? stream))
{
_logger.Info($"Cache hit {asset.url}");
return stream;
}
var response = await _client.GetAsync(asset.url);
_logger.Info($"Got response {response.StatusCode}");
var stream = await response.Content.ReadAsStreamAsync();
stream = await response.Content.ReadAsStreamAsync();
_cache.Set(asset.url, stream);
return stream;
}
}

12
Elementary/Audio/VoicevoxAPI.cs
View File

@ -1,5 +1,6 @@
using System.Net.Http.Json;
using System.Text;
using Microsoft.Extensions.Caching.Memory;
using NLog;
namespace Elementary.Audio;
@ -9,6 +10,7 @@ public class VoicevoxAPI
private UriBuilder _APIRootUrl;
private HttpClient _client;
private ILogger _logger;
private MemoryCache _cache;
public async Task Setup(string url)
{
@ -17,6 +19,7 @@ public class VoicevoxAPI
_APIRootUrl = new UriBuilder($"{_url.Scheme}://{_url.Host}:{_url.Port}");
_client = new HttpClient();
_logger = LogManager.GetCurrentClassLogger();
_cache = new MemoryCache(new MemoryCacheOptions());
}
/// <summary>
@ -29,10 +32,17 @@ public class VoicevoxAPI
{
_logger.Info($"Requested TTS {text}");
if (_cache.TryGetValue(text, out Stream? stream))
{
_logger.Info($"Cache hit {text}");
return stream;
}
var query = await GetAudioQuery(text, speaker);
if (query == null) return null;
var stream = await GetAudioStream(query, speaker);
stream = await GetAudioStream(query, speaker);
if (stream == null) return null;
_cache.Set(text, stream);
return stream;
}

39
Elementary/Audio/Ytdlp.cs Normal file
View File

@ -0,0 +1,39 @@
using System.Diagnostics;
using System.Text;
using NLog;
namespace Elementary.Audio;
public class Ytdlp
{
private readonly ILogger _logger;
public Ytdlp()
{
_logger = LogManager.GetCurrentClassLogger();
}
public async Task<Stream> GetStream(string url)
{
var process = new Process
{
StartInfo =
{
FileName = "yt-dlp",
Arguments = $"-f bestaudio -o - {url}",
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
},
EnableRaisingEvents = true
};
var stream = new MemoryStream();
process.Start();
process.StandardOutput.BaseStream.CopyTo(stream);
process.WaitForExit();
stream.Position = 0;
return stream;
}
}

7
Elementary/Commands/MessageCommands.cs
View File

@ -112,4 +112,11 @@ public class MessageCommands : ModuleBase<SocketCommandContext>
result += "```";
return ReplyAsync(result);
}
[Command("ytdlp", RunMode = RunMode.Async)]
[Summary("Play sound from Youtube URL.")]
public async Task YtdlpAsync([Summary("Youtube URL")] string url)
{
await _audioManager.PlayYoutube(url);
}
}

7
Elementary/Config/Configuration.cs
View File

@ -10,6 +10,13 @@ public class AppSettings
public SozaiSettings SozaiSettings { get; set; }
public EmojiSettings EmojiSettings { get; set; }
public VoicevoxSettings VoicevoxSettings { get; set; }
public MecabSettings MecabSettings { get; set; }
}
public class MecabSettings
{
public bool enabled { get; set; }
public string? dictionaryPath { get; set; }
}
public class VoicevoxSettings

67
Elementary/Dictionary/Mecab.cs Normal file
View File

@ -0,0 +1,67 @@
using System.Diagnostics;
using System.Text;
using System.Text.RegularExpressions;
namespace Elementary.Dictionary;
public class Mecab
{
private string? _dictionaryPath;
private bool _enabled;
public async Task Setup(bool enabled, string? dictionaryPath)
{
_dictionaryPath = dictionaryPath;
_enabled = enabled;
await Task.CompletedTask;
}
public async Task<String> ParseToKana(string text)
{
if (!_enabled)
{
return text;
}
// "Apple PencilあああiPad" -> "Apple Pencil iPad"
var englishWords = Regex.Matches(text, @"[a-zA-Z]+").Select(m => m.Value).ToList();
var englishWordsString = string.Join(" ", englishWords);
String args = $"-Oyomi";
if (_dictionaryPath != null)
{
args += $" -d {_dictionaryPath}";
}
var process = new Process
{
StartInfo =
{
FileName = "mecab",
Arguments = args,
RedirectStandardInput = true,
RedirectStandardOutput = true,
StandardOutputEncoding = Encoding.UTF8,
StandardInputEncoding = Encoding.UTF8,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
}
};
process.Start();
await process.StandardInput.WriteLineAsync(englishWordsString);
await process.StandardInput.FlushAsync();
process.StandardInput.Close();
var result = await process.StandardOutput.ReadToEndAsync();
process.WaitForExit();
// replace english words with kana
var kanaWords = result.Split(" ").Select(m => m.Trim()).ToList();
for (var i = 0; i < englishWords.Count; i++)
{
text = text.Replace(englishWords[i], kanaWords[i]);
}
return text;
}
}

4
Elementary/Elementary.csproj
View File

@ -17,12 +17,10 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="CSCore" Version="1.2.1.2"/>
<PackageReference Include="Discord.Net" Version="3.11.0"/>
<PackageReference Include="Discord.Net.Core" Version="3.11.0"/>
<PackageReference Include="LiteDB" Version="5.0.17" />
<PackageReference Include="ManagedBass" Version="3.1.1"/>
<PackageReference Include="ManagedBass.Mix" Version="3.1.1"/>
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0"/>
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.4"/>
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="7.0.0"/>

9
Elementary/Program.cs
View File

@ -7,7 +7,6 @@ using Discord.WebSocket;
using Elementary.Audio;
using Elementary.Commands;
using Elementary.Dictionary;
using ManagedBass;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using NAudio.Wave;
@ -30,6 +29,8 @@ public class Program
private VoicevoxAPI _voicevoxAPI;
private EmojiDictionary _emojiDictionary;
private DictionaryDB _dictionaryDB;
private Mecab _mecab;
private Ytdlp _ytdlp;
public static async Task Main(string[] args)
{
@ -73,15 +74,19 @@ public class Program
.AddSingleton<MessageHandler>()
.AddSingleton<EmojiDictionary>()
.AddSingleton<DictionaryDB>()
.AddSingleton<Mecab>()
.AddSingleton<Ytdlp>()
.BuildServiceProvider();
_logger = LogManager.GetCurrentClassLogger();
_sozaiAPI = _services.GetRequiredService<SozaiAPI>();
_voicevoxAPI = _services.GetRequiredService<VoicevoxAPI>();
_ytdlp = _services.GetRequiredService<Ytdlp>();
_emojiDictionary = _services.GetRequiredService<EmojiDictionary>();
_dictionaryDB = _services.GetRequiredService<DictionaryDB>();
_mecab = _services.GetRequiredService<Mecab>();
_handler = new(_client, _commands, _services, _services.GetRequiredService<AudioManager>(),
_services.GetRequiredService<MessageHandler>());
@ -96,6 +101,8 @@ public class Program
await _voicevoxAPI.Setup(configuration.AppSettings.VoicevoxSettings.Url);
await _emojiDictionary.Setup(configuration.AppSettings.EmojiSettings.DictionaryPath);
await _dictionaryDB.Setup();
await _mecab.Setup(configuration.AppSettings.MecabSettings.enabled,
configuration.AppSettings.MecabSettings.dictionaryPath);
await _client.LoginAsync(TokenType.Bot, configuration.AppSettings.DiscordSettings.Token);
await _client.StartAsync();
await _client.SetActivityAsync(new Game("!join"));

3
Elementary/appsettings.json
View File

@ -18,6 +18,9 @@
},
"EmojiSettings": {
"DictionaryPath": "emoji-ja\\data\\emoji_ja.json"
},
"MecabSettings": {
"Enabled": true
}
}
}