Jordan/PEAK-JordanMod
1
0
Fork 0
mirror of https://github.com/TheJordanDev/PEAK-JordanMod.git synced 2026-10-16 23:56:44 +02:00
Code Issues Packages Projects Releases Wiki Activity

0.1.15 | BingBong live voice (client groundwork)

Browse Source 
Lets a live microphone be streamed through BingBong from a relay rather
than over PEAK's Photon servers. This is the in-game client only: it
connects over a WebSocket, decodes the audio and plays it. The relay and
the web app that captures the microphone are not built yet, so the
feature does nothing on its own.

Playback uses OnAudioFilterRead rather than a streaming AudioClip. A clip
created with stream:true fills Unity's decode buffer ahead of playback,
and that read-ahead cannot be satisfied from a live stream because the
audio has not arrived yet -- the buffer drained to empty on a cycle no
matter how deep it was made, which was audible as constant clicking. The
filter callback runs in lockstep with playback, so demand can never
outrun real time.

Includes tools/stub_relay.py, a development stub speaking the same wire
format, so the client can be tested end to end without the real backend,
a microphone or a browser.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
TheJordan
2026-09-05 16:18:04 +02:00
parent 50a7845345
commit 0d2350553a
8 changed files with 1137 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
CHANGELOG.md
View File

@@ -94,3 +94,12 @@
- The audio bank sync no longer drives `UnityWebRequest` from a background thread, which is main-thread only, and no longer busy-waits on it while blocking a second thread-pool thread.
- The song display no longer scans every loaded object looking for its font on every frame. It falls back to the default font immediately and retries for the nicer one on an interval, stopping once found.
- The audio bank URL is normalised before use: a missing scheme is filled in (http for localhost, https otherwise) and a trailing slash trimmed. Without a scheme every request failed outright with "Invalid BugleSoundAPIURL in config", since the URL could not be parsed.
# v0.1.15 | BingBong live voice (work in progress)
- Groundwork for speaking through BingBong with a live microphone, streamed from a relay rather than over PEAK's Photon servers. This release contains the in-game client only: it connects to a relay over a WebSocket, decodes the incoming audio and plays it. The relay itself and the web app that captures the microphone are not built yet, so the feature does nothing on its own.
- New "BingBong Voice" config section. The relay URL is derived from `BugleSoundAPIURL` when left empty, since that is where the relay will live, and can be overridden to point anywhere (a bare host or `host:port` is accepted and completed automatically).
- Playback goes through `OnAudioFilterRead` rather than a streaming `AudioClip`. A clip created with `stream: true` fills Unity's decode buffer *ahead* of playback, and that read-ahead cannot be satisfied from a live stream because the audio has not arrived yet — the buffer drained to empty on a cycle no matter how deep it was, which was audible as constant clicking. The filter callback runs in lockstep with playback instead, so demand can never outrun real time.
- Incoming audio is resampled to the output rate by the mod rather than relying on Unity's mixer, so a device running at 44.1kHz or 48kHz sounds identical. Verified at every rate combination.
- A jitter buffer absorbs network timing, with a shortfall rendering what it has and padding the rest rather than muting and refilling, which pumped audibly when it happened often.
- `tools/stub_relay.py` is a development stub of the relay, enough to test the client end to end before the real backend exists. It speaks the same wire format and can generate a test tone, so no microphone or browser is needed.

67
src/JordanMod/ConfigHandler.cs
View File

@@ -29,6 +29,16 @@ public static class ConfigHandler
public static ConfigEntry<string> FavoriteSongsList { get; private set; } = null!;
public static ConfigEntry<KeyCode> FavoriteSongToggleKey { get; private set; } = null!;
// BingBong Voice settings
public static ConfigEntry<bool> BingBongVoiceEnabled { get; private set; } = null!;
public static ConfigEntry<string> BingBongVoiceRelayURL { get; private set; } = null!;
public static ConfigEntry<string> BingBongVoiceDevToken { get; private set; } = null!;
public static ConfigEntry<float> BingBongVoiceVolume { get; private set; } = null!;
public static ConfigEntry<float> BingBongVoiceDuckVolume { get; private set; } = null!;
public static ConfigEntry<float> BingBongVoiceDuckLead { get; private set; } = null!;
public static ConfigEntry<float> BingBongVoiceJitterMs { get; private set; } = null!;
public static ConfigEntry<KeyCode> BingBongVoiceDebugKey { get; private set; } = null!;
public static void Initialize(ConfigFile configFile)
{
Config = configFile;
@@ -107,6 +117,63 @@ public static class ConfigHandler
new ConfigDescription("Key to toggle favorite status of current song")
);
// BingBong Voice settings
BingBongVoiceEnabled = Config.Bind(
"BingBong Voice",
"Enabled",
true,
new ConfigDescription("Let BingBong speak with a live voice streamed from the relay. When off, no patches are applied at all.")
);
BingBongVoiceRelayURL = Config.Bind(
"BingBong Voice",
"RelayURL",
"",
new ConfigDescription("Override for the voice relay WebSocket URL. Leave empty to derive it from BugleSoundAPIURL, which is where the relay lives by default (https://host -> wss://host/voice/ws).")
);
BingBongVoiceDevToken = Config.Bind(
"BingBong Voice",
"DevToken",
"listen:test",
new ConfigDescription("Temporary token appended to the relay URL when it carries none. The real join flow will fetch this from the audio bank instead.")
);
BingBongVoiceVolume = Config.Bind(
"BingBong Voice",
"Volume",
1.0f,
new ConfigDescription("Live voice volume", new AcceptableValueRange<float>(0f, 1f))
);
BingBongVoiceDuckVolume = Config.Bind(
"BingBong Voice",
"DuckVolume",
0.0f,
new ConfigDescription("Live voice volume while one of BingBong's own voice lines is playing", new AcceptableValueRange<float>(0f, 1f))
);
BingBongVoiceDuckLead = Config.Bind(
"BingBong Voice",
"DuckLeadSeconds",
1.0f,
new ConfigDescription("How long before a voice line's audio actually starts to begin ducking. AskRoutine waits about a second before it plays anything.", new AcceptableValueRange<float>(0f, 2f))
);
BingBongVoiceJitterMs = Config.Bind(
"BingBong Voice",
"JitterBufferMs",
80f,
new ConfigDescription("Audio buffered before playback starts. Lower is more responsive, higher survives worse connections.", new AcceptableValueRange<float>(40f, 400f))
);
BingBongVoiceDebugKey = Config.Bind(
"BingBong Voice",
"DebugToggleKey",
KeyCode.F7,
new ConfigDescription("Connect/disconnect the relay and play it through a non-positional debug source, for testing without a BingBong")
);
}
}

2
src/JordanMod/JordanMod.csproj
View File

@@ -8,7 +8,7 @@
<!-- This is the display name of your mod. Example: BepInEx Template -->
<AssemblyTitle>JordanMod</AssemblyTitle>
<!-- This is the version number of your mod. -->
<Version>0.1.14</Version>
<Version>0.1.15</Version>
</PropertyGroup>
<PropertyGroup>

236
src/JordanMod/modules/bingbong_voice/BingBongVoiceClient.cs Normal file
View File

@@ -0,0 +1,236 @@
using System;
using System.Security.Authentication;
using System.Threading;
using WebSocketSharp;
namespace JordanMod.Modules.BingBongVoice;
/// One decoded stream. Swapped wholesale when the sender changes sample rate, so consumers
/// only ever hold a consistent (ring, rate) pair.
public sealed class VoiceStream
{
public VoiceRingBuffer Ring { get; }
public int SampleRate => Ring.SampleRate;
public VoiceStream(int sampleRate) => Ring = new VoiceRingBuffer(sampleRate);
}
/// <summary>
/// Holds the websocket to the voice relay and turns its binary frames into samples in a ring.
///
/// Everything here runs on websocket-sharp's own receive thread. It must never touch a Unity
/// API: hop through <see cref="Plugin.RunOnMainThread"/> for that, and never per packet — at
/// 50 frames a second that queue would balloon during any frame hitch.
/// </summary>
public sealed class BingBongVoiceClient
{
// Wire format, kept in step with the plan and the stub relay:
// 0 u8 magic | 1 u8 version | 2 u8 codec | 3 u8 rate code | 4 u32 seq | 8 u32 sampleTimestamp
private const int HeaderSize = 12;
private const byte Magic = 0xB1;
private const byte Version = 1;
private const byte CodecPcm16 = 0;
private static readonly int[] RateTable = [16000, 8000, 24000, 48000];
// A gap larger than this is treated as a restart rather than something to paper over.
private const int MaxGapSamples = 48000;
private WebSocket? _socket;
private VoiceStream? _stream;
private Thread? _reconnectThread;
private volatile bool _shouldRun;
private volatile bool _speakerActive;
private long _expectedTimestamp = -1;
private int _badFrames;
/// The live stream, or null before the first frame arrives. Read from the audio thread.
public VoiceStream? Stream => Volatile.Read(ref _stream);
/// True between speaker_joined and speaker_left control frames.
public bool SpeakerActive => _speakerActive;
public bool IsConnected => _socket is { ReadyState: WebSocketState.Open };
public string Url { get; private set; } = "";
public void Connect(string url)
{
if (_shouldRun) return;
if (string.IsNullOrWhiteSpace(url))
{
Debug.LogWarning("[BingBongVoice] No relay URL configured, not connecting.");
return;
}
// A malformed URL will never start working, so fail here rather than letting the
// reconnect loop retry it every minute for the rest of the session.
if (!Uri.TryCreate(url, UriKind.Absolute, out Uri? parsed) || (parsed.Scheme != "ws" && parsed.Scheme != "wss"))
{
Debug.LogError($"[BingBongVoice] '{url}' is not a usable WebSocket URL, not connecting.");
return;
}
Url = url;
_shouldRun = true;
_reconnectThread = new Thread(ConnectLoop) { IsBackground = true, Name = "BingBongVoice" };
_reconnectThread.Start();
}
public void Disconnect()
{
_shouldRun = false;
_speakerActive = false;
WebSocket? socket = _socket;
_socket = null;
if (socket == null) return;
try { socket.Close(CloseStatusCode.Normal, "disconnecting"); }
catch (Exception e) { Debug.LogWarning($"[BingBongVoice] Error closing socket: {e.Message}"); }
}
// Reconnect with exponential backoff. Runs on its own thread for the whole session.
private void ConnectLoop()
{
int attempt = 0;
System.Random jitter = new();
while (_shouldRun)
{
try
{
OpenSocket();
// Poll rather than block: websocket-sharp gives us no "wait until closed".
while (_shouldRun && _socket is { ReadyState: WebSocketState.Open }) Thread.Sleep(200);
if (!_shouldRun) return;
attempt++;
}
catch (Exception e)
{
attempt++;
// First failure and then every tenth, so an unreachable relay can't flood the log.
if (attempt == 1 || attempt % 10 == 0)
Debug.LogWarning($"[BingBongVoice] Connection attempt {attempt} failed: {e.Message}");
}
if (!_shouldRun) return;
int delayMs = (int)(Math.Min(60000, 2000 * Math.Pow(2, Math.Min(attempt - 1, 5))) * (0.8 + jitter.NextDouble() * 0.4));
for (int slept = 0; slept < delayMs && _shouldRun; slept += 100) Thread.Sleep(100);
}
}
private void OpenSocket()
{
WebSocket socket = new(Url);
// websocket-sharp writes its own "Fatal|WebSocket.connect|..." lines straight to the
// console, including a run of mojibake, every single reconnect attempt. We report
// connection state ourselves, so silence its logger entirely.
socket.Log.Output = (_, _) => { };
// This build of websocket-sharp defaults to a legacy protocol set, so wss:// to any
// modern TLS terminator fails outright without this.
if (Url.StartsWith("wss://", StringComparison.OrdinalIgnoreCase))
{
socket.SslConfiguration.EnabledSslProtocols =
SslProtocols.Tls12 | SslProtocols.Tls11 | SslProtocols.Tls;
}
socket.OnMessage += OnMessage;
socket.OnOpen += (_, _) =>
{
_expectedTimestamp = -1;
_badFrames = 0;
Debug.Log("[BingBongVoice] Connected to relay.");
};
socket.OnClose += (_, e) =>
{
_speakerActive = false;
Debug.Log($"[BingBongVoice] Relay closed ({e.Code} {e.Reason}).");
};
socket.OnError += (_, e) => Debug.LogWarning($"[BingBongVoice] Socket error: {e.Message}");
_socket = socket;
socket.Connect();
}
// websocket-sharp's receive thread. No Unity APIs, no allocation beyond what we're handed.
private void OnMessage(object? sender, MessageEventArgs e)
{
try
{
if (!e.IsBinary)
{
HandleControl(e.Data);
return;
}
byte[] d = e.RawData;
if (d.Length < HeaderSize) return;
if (d[0] != Magic || d[1] != Version)
{
if (++_badFrames <= 3) Debug.LogWarning($"[BingBongVoice] Bad frame header (magic {d[0]:X2}, version {d[1]}).");
return;
}
if (d[2] != CodecPcm16)
{
if (++_badFrames <= 3) Debug.LogWarning($"[BingBongVoice] Unsupported codec {d[2]}, frame dropped.");
return;
}
int rate = RateTable[d[3] & 3];
uint timestamp = (uint)(d[8] | (d[9] << 8) | (d[10] << 16) | (d[11] << 24));
VoiceStream stream = EnsureStream(rate);
// The sender's timestamp tells us exactly how much audio went missing, so a dropped
// frame costs its own duration instead of silently shifting everything after it.
if (_expectedTimestamp >= 0 && timestamp != _expectedTimestamp)
{
long gap = (long)timestamp - _expectedTimestamp;
if (gap > 0 && gap <= MaxGapSamples) stream.Ring.WriteSilence((int)gap);
// A negative or huge gap means the sender restarted; just carry on from here.
}
int payload = d.Length - HeaderSize;
stream.Ring.WritePcm16(d, HeaderSize, payload);
_expectedTimestamp = timestamp + (payload >> 1);
}
catch (Exception ex)
{
if (++_badFrames <= 3) Debug.LogError($"[BingBongVoice] Frame handling failed: {ex}");
}
}
private VoiceStream EnsureStream(int rate)
{
VoiceStream? current = Volatile.Read(ref _stream);
if (current != null && current.SampleRate == rate) return current;
VoiceStream replacement = new(rate);
Volatile.Write(ref _stream, replacement);
_expectedTimestamp = -1;
Debug.Log($"[BingBongVoice] Stream rate is {rate} Hz.");
return replacement;
}
private void HandleControl(string json)
{
// Deliberately not parsing JSON properly: the only fields that matter are these two
// markers, and a substring check can't throw on a malformed payload.
if (json.Contains("speaker_joined"))
{
_speakerActive = true;
Debug.Log("[BingBongVoice] A speaker joined.");
}
else if (json.Contains("speaker_left") || json.Contains("preempted"))
{
_speakerActive = false;
_expectedTimestamp = -1;
Debug.Log("[BingBongVoice] The speaker left.");
}
}
}

171
src/JordanMod/modules/bingbong_voice/BingBongVoiceModule.cs Normal file
View File

@@ -0,0 +1,171 @@
using System;
using UnityEngine;
namespace JordanMod.Modules.BingBongVoice;
[Module(Enabled = true)]
class BingBongVoiceModule : Module
{
public static BingBongVoiceModule? Instance { get; private set; }
public override string ModuleName => "BingBong Voice Module";
public BingBongVoiceClient? Client { get; private set; }
private GameObject? _debugObject;
public override Type[] GetPatches()
{
// Config is bound before modules load (Plugin.Awake: ConfigHandler.Initialize ->
// SetupModules -> SetupPatches), so turning the feature off here means the Harmony
// patches are never applied at all and the mod has zero footprint.
if (!ConfigHandler.BingBongVoiceEnabled.Value) return [];
return [];
}
public override void Initialize()
{
if (Instance != null) return;
Instance = this;
base.Initialize();
}
public override void Update()
{
if (!ConfigHandler.BingBongVoiceEnabled.Value) return;
if (Input.GetKeyDown(ConfigHandler.BingBongVoiceDebugKey.Value)) ToggleDebugStream();
}
/// <summary>
/// Where the relay lives. The relay is served by the audio bank, so an unset override just
/// derives the WebSocket URL from BugleSoundAPIURL; setting it points somewhere else
/// entirely (a local stub, a separate host).
/// </summary>
public static string ResolveRelayUrl()
{
string overrideUrl = (ConfigHandler.BingBongVoiceRelayURL.Value ?? "").Trim();
string source;
string url;
if (overrideUrl.Length > 0)
{
// An override names the endpoint. Only fill in /voice/ws when it has no path at all,
// so "localhost:8787" works but "wss://host/custom/path" is left alone.
source = overrideUrl;
url = NormalizeOverride(overrideUrl);
}
else
{
// The audio bank URL names a service root, which may itself sit under a path
// (https://audiobank.thejordan.dev/api), so the endpoint always goes on the end.
source = (ConfigHandler.BugleSoundAPIURL.Value ?? "").Trim();
if (source.Length == 0)
{
Debug.LogWarning("[BingBongVoice] Neither RelayURL nor BugleSoundAPIURL is set, nowhere to connect.");
return "";
}
url = DeriveFromApiBase(source);
}
if (url.Length == 0)
{
Debug.LogError($"[BingBongVoice] Could not make a usable WebSocket URL out of '{source}'. Expected something like 'ws://localhost:8787/voice/ws' or just 'localhost:8787'.");
return "";
}
// Until the real /voice/join flow lands, the token comes from config rather than a
// lookup. A URL that already carries one is left alone.
if (url.IndexOf("token=", StringComparison.OrdinalIgnoreCase) < 0)
{
string token = (ConfigHandler.BingBongVoiceDevToken.Value ?? "").Trim();
if (token.Length > 0)
url += (url.IndexOf('?') >= 0 ? "&" : "?") + "token=" + Uri.EscapeDataString(token);
}
return url;
}
private static string NormalizeOverride(string raw)
{
string url = ToWebSocketScheme(raw);
// Split the query/fragment off first so appending a path can't mangle them.
int split = url.IndexOfAny(['?', '#']);
string tail = split >= 0 ? url.Substring(split) : "";
string head = split >= 0 ? url.Substring(0, split) : url;
if (!IsWebSocketUrl(head, out Uri? parsed)) return "";
// A bare host means the caller gave us the root, not the endpoint.
if (parsed.AbsolutePath.Length <= 1) head = head.TrimEnd('/') + "/voice/ws";
return head + tail;
}
private static string DeriveFromApiBase(string apiBase)
{
string url = ToWebSocketScheme(apiBase);
// The base is a service root, so anything after it is not ours to keep.
int split = url.IndexOfAny(['?', '#']);
if (split >= 0) url = url.Substring(0, split);
url = url.TrimEnd('/') + "/voice/ws";
return IsWebSocketUrl(url, out _) ? url : "";
}
private static bool IsWebSocketUrl(string url, out Uri parsed)
{
parsed = null!;
if (!Uri.TryCreate(url, UriKind.Absolute, out Uri? candidate)) return false;
if (candidate.Scheme != "ws" && candidate.Scheme != "wss") return false;
parsed = candidate;
return true;
}
private static string ToWebSocketScheme(string url)
{
if (url.StartsWith("ws://", StringComparison.OrdinalIgnoreCase)) return url;
if (url.StartsWith("wss://", StringComparison.OrdinalIgnoreCase)) return url;
if (url.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) return "wss://" + url.Substring(8);
if (url.StartsWith("http://", StringComparison.OrdinalIgnoreCase)) return "ws://" + url.Substring(7);
// No scheme at all: same rule as everywhere else, loopback gets the insecure variant.
return Helper.EnsureScheme(url, "wss", "ws");
}
// Stage 1 harness: connect and play through a non-positional source, so the whole network
// and audio path can be tested with no BingBong, no lobby and no browser.
private void ToggleDebugStream()
{
if (Client != null)
{
Debug.Log("[BingBongVoice] Stopping debug stream.");
Client.Disconnect();
Client = null;
if (_debugObject != null) UnityEngine.Object.Destroy(_debugObject);
_debugObject = null;
return;
}
string url = ResolveRelayUrl();
if (url.Length == 0) return;
Debug.Log($"[BingBongVoice] Starting debug stream against {url}");
Client = new BingBongVoiceClient();
Client.Connect(url);
_debugObject = new GameObject("BingBongVoiceDebug");
UnityEngine.Object.DontDestroyOnLoad(_debugObject);
BingBongVoiceSource.Attach(_debugObject, null);
}
public override void Destroy()
{
Client?.Disconnect();
Client = null;
if (_debugObject != null) UnityEngine.Object.Destroy(_debugObject);
_debugObject = null;
base.Destroy();
}
}

296
src/JordanMod/modules/bingbong_voice/BingBongVoiceSource.cs Normal file
View File

@@ -0,0 +1,296 @@
using System;
using UnityEngine;
namespace JordanMod.Modules.BingBongVoice;
/// <summary>
/// Plays the live voice stream through one AudioSource. One of these per BingBong (and one
/// non-positional debug instance), each with its own read cursor into the shared ring.
///
/// Output goes through <c>OnAudioFilterRead</c> rather than a procedural streaming AudioClip.
/// A clip created with <c>stream: true</c> fills Unity's own decode buffer ahead of playback --
/// that read-ahead is by design, and it cannot be satisfied from a live stream because the audio
/// simply has not arrived yet, so the ring drained to empty no matter how deep the jitter buffer
/// was made. OnAudioFilterRead is invoked exactly once per DSP block, in lockstep with playback,
/// so demand can never outrun real time. The AudioSource still needs *something* playing for the
/// filter to run, hence the silent carrier clip.
/// </summary>
public sealed class BingBongVoiceSource : MonoBehaviour
{
private enum State { Buffering, Playing }
private AudioSource? _source;
private AudioClip? _carrier;
private VoiceStream? _stream;
private int _outputRate;
private double _read; // fractional position into the ring, audio thread only
private State _state = State.Buffering;
private float _gain; // smoothed toward _targetGain, audio thread only
private volatile float _targetGain;
private volatile float _amplitude;
private volatile bool _dead;
private int _underruns;
private float _lastReportTime;
private int _strikes;
private volatile int _depthSamples;
private volatile int _minDepth = int.MaxValue;
private volatile int _streamRate = 1;
private volatile int _blockFrames; // frames per DSP block, as actually delivered
private static bool _reportedBlock; // once per session, not once per BingBong
private float _gainStep = 0.0008f;
private long _lastWritePos;
private float _lastRateTime;
/// Set by the duck patch. Live voice is attenuated until this time.
public float DuckUntil { get; set; }
public float Amplitude => _amplitude;
public static BingBongVoiceSource Attach(GameObject target, AudioSource? template)
{
if (target.TryGetComponent(out BingBongVoiceSource existing)) return existing;
BingBongVoiceSource voice = target.AddComponent<BingBongVoiceSource>();
voice.Configure(template);
return voice;
}
private void Configure(AudioSource? template)
{
_outputRate = AudioSettings.outputSampleRate;
_gainStep = 1f / (_outputRate * 0.015f); // ~15ms attack whatever the device runs at
// Silent carrier: the filter only runs while the source is playing something. Its
// contents are irrelevant, OnAudioFilterRead overwrites the buffer wholesale.
_carrier = AudioClip.Create($"BingBongVoiceCarrier_{GetInstanceID()}", 4096, 1, _outputRate, false);
_source = gameObject.AddComponent<AudioSource>();
_source.clip = _carrier;
_source.loop = true;
_source.playOnAwake = false;
if (template != null)
{
// Copy the vanilla voice-line source's 3D setup rather than guessing, so the live
// voice attenuates exactly like BingBong's own lines. Routing through its mixer
// group also means the game's volume sliders apply for free.
_source.spatialBlend = template.spatialBlend;
_source.minDistance = template.minDistance;
_source.maxDistance = template.maxDistance;
_source.rolloffMode = template.rolloffMode;
_source.dopplerLevel = template.dopplerLevel;
_source.spread = template.spread;
_source.outputAudioMixerGroup = template.outputAudioMixerGroup;
}
else
{
_source.spatialBlend = 0f; // debug instance: non-positional, always audible
}
_source.Play();
AudioSettings.OnAudioConfigurationChanged += OnAudioConfigChanged;
}
private void OnAudioConfigChanged(bool deviceChanged)
{
// Owning the resample ratio ourselves makes this a one-line update instead of a rebuild.
_outputRate = AudioSettings.outputSampleRate;
_gainStep = 1f / (_outputRate * 0.015f);
_state = State.Buffering;
}
private void Update()
{
_stream = BingBongVoiceModule.Instance?.Client?.Stream;
_targetGain = Time.time < DuckUntil
? ConfigHandler.BingBongVoiceDuckVolume.Value
: ConfigHandler.BingBongVoiceVolume.Value;
if (!_reportedBlock && _blockFrames > 0)
{
_reportedBlock = true;
AudioSettings.GetDSPBufferSize(out int dspLength, out int dspCount);
Debug.Log($"[BingBongVoice] DSP block is {_blockFrames} frames ({_blockFrames * 1000 / Math.Max(1, _outputRate)}ms at {_outputRate}Hz); buffer {dspLength}x{dspCount}.");
}
if (Time.time - _lastReportTime > 5f && _underruns > 0)
{
_lastReportTime = Time.time;
int depthMs = _depthSamples * 1000 / Math.Max(1, _streamRate);
int minMs = _minDepth == int.MaxValue ? -1 : _minDepth * 1000 / Math.Max(1, _streamRate);
int targetMs = TargetSamples(_streamRate) * 1000 / Math.Max(1, _streamRate);
long writeNow = _stream?.Ring.WritePosition ?? 0;
float elapsed = Time.time - _lastRateTime;
float producedRatio = elapsed > 0.01f && _lastWritePos > 0
? (writeNow - _lastWritePos) / (elapsed * Math.Max(1, _streamRate))
: 1f;
_lastWritePos = writeNow;
_lastRateTime = Time.time;
string hint = producedRatio < 0.95f
? $" - SENDER is only producing {producedRatio:P0} of real time"
: depthMs <= 1 ? " - CONSUMER is draining faster than real time" : "";
Debug.Log($"[BingBongVoice] buffer health: {_underruns} underruns, depth {depthMs}ms (min {minMs}ms) of {targetMs}ms{hint}");
_underruns = 0;
_minDepth = int.MaxValue;
}
}
// How much audio to hold before playing. One DSP block is consumed on every call, so the
// floor is (target - block); keeping two blocks plus the configured margin leaves that floor
// comfortably above the point where a read cannot be satisfied.
private int TargetSamples(int streamRate)
{
int blockStream = (int)(_blockFrames * (double)streamRate / Math.Max(1, _outputRate)) + 1;
return blockStream * 2 + (int)(streamRate * ConfigHandler.BingBongVoiceJitterMs.Value / 1000f);
}
// ---- audio thread from here down -------------------------------------------------
// Allocation-free, lock-free, no Unity API calls, no logging. An exception escaping onto
// the DSP thread is not recoverable, so everything is wrapped and falls back to silence.
private void OnAudioFilterRead(float[] data, int channels)
{
try
{
Fill(data, Math.Max(1, channels));
}
catch
{
Array.Clear(data, 0, data.Length);
_amplitude = 0f;
if (++_strikes >= 3) _dead = true;
}
}
private void Fill(float[] data, int channels)
{
int frames = data.Length / channels;
_blockFrames = frames;
VoiceStream? stream = _stream;
if (_dead || stream == null)
{
Silence(data);
return;
}
VoiceRingBuffer ring = stream.Ring;
long write = ring.WritePosition;
double ratio = (double)ring.SampleRate / _outputRate;
int targetSamples = TargetSamples(ring.SampleRate);
if (_state == State.Buffering)
{
if (write - (long)_read < targetSamples || write < targetSamples)
{
Silence(data);
return;
}
_read = write - targetSamples;
_state = State.Playing;
}
// Drifted too far behind (sender burst, or we were paused). Rejoin the live edge rather
// than playing further and further into the past.
if (write - (long)_read > targetSamples * 5)
{
_read = write - targetSamples;
}
double needed = frames * ratio;
long available = write - (long)_read;
_depthSamples = (int)Math.Min(int.MaxValue, available);
if (_depthSamples < _minDepth) _minDepth = _depthSamples;
_streamRate = ring.SampleRate;
int usableFrames = frames;
if (available < needed + 2)
{
usableFrames = (int)Math.Max(0, (available - 2) / ratio);
if (usableFrames > frames) usableFrames = frames;
_underruns++;
// Only rebuild the whole buffer when there was essentially nothing there. A brief
// shortfall should cost a few samples, not a fade plus a full buffer of silence.
if (usableFrames < frames / 4) _state = State.Buffering;
}
Render(data, ring, channels, usableFrames, ratio, fadeOut: usableFrames < frames);
// Pad any frames we could not fill.
for (int f = usableFrames; f < frames; f++)
for (int c = 0; c < channels; c++)
data[f * channels + c] = 0f;
}
private void Render(float[] data, VoiceRingBuffer ring, int channels, int frames, double ratio, bool fadeOut)
{
float peak = 0f;
float fadeStep = fadeOut && frames > 0 ? _gain / frames : 0f;
for (int f = 0; f < frames; f++)
{
long i0 = (long)_read;
float frac = (float)(_read - i0);
float sample = ring[i0] * (1f - frac) + ring[i0 + 1] * frac;
if (fadeOut) _gain = Math.Max(0f, _gain - fadeStep);
else _gain += (_targetGain - _gain) * _gainStep;
float outSample = sample * _gain;
int baseIndex = f * channels;
for (int c = 0; c < channels; c++) data[baseIndex + c] = outSample;
float abs = outSample < 0f ? -outSample : outSample;
if (abs > peak) peak = abs;
_read += ratio;
}
_amplitude = peak;
}
private void Silence(float[] data)
{
Array.Clear(data, 0, data.Length);
_gain = 0f;
_amplitude = 0f;
}
// ----------------------------------------------------------------------------------
private void OnDestroy()
{
// The audio thread can and will run the filter after Destroy happens on the main thread,
// so flag it dead first: that check is what makes the teardown safe.
_dead = true;
AudioSettings.OnAudioConfigurationChanged -= OnAudioConfigChanged;
if (_source != null)
{
_source.Stop();
_source.clip = null;
}
if (_carrier != null) Destroy(_carrier);
}
private void OnDisable()
{
if (_source != null) _source.Stop();
_state = State.Buffering;
}
private void OnEnable()
{
// Rejoin live rather than trying to resume where we left off.
_state = State.Buffering;
if (_source != null && _carrier != null) _source.Play();
}
}

71
src/JordanMod/modules/bingbong_voice/VoiceRingBuffer.cs Normal file
View File

@@ -0,0 +1,71 @@
using System;
using System.Threading;
namespace JordanMod.Modules.BingBongVoice;
/// <summary>
/// Ring of mono float samples at the stream's sample rate. Single producer (the websocket
/// receive thread), multiple consumers (one <see cref="BingBongVoiceSource"/> per BingBong,
/// each reading from the audio thread with its own cursor).
///
/// Lock-free by construction: the producer publishes <see cref="WritePosition"/> only after the
/// samples are in place, and the ring is far deeper than the jitter target, so a consumer can
/// never observe a slot that is being overwritten under it. Consumers hold absolute positions
/// and resync if they ever fall further behind than the ring is long.
/// </summary>
public sealed class VoiceRingBuffer
{
private readonly float[] _buf;
private readonly int _mask;
private long _write;
public int SampleRate { get; }
public int Capacity => _buf.Length;
/// Total samples ever written. Monotonic; never reset, so consumer cursors stay meaningful.
public long WritePosition => Volatile.Read(ref _write);
public VoiceRingBuffer(int sampleRate, float seconds = 4f)
{
SampleRate = sampleRate;
int wanted = Math.Max(1024, (int)(sampleRate * seconds));
int size = 1;
while (size < wanted) size <<= 1; // power of two so the wrap is a mask, not a modulo
_buf = new float[size];
_mask = size - 1;
}
/// Producer thread only. Converts interleaved little-endian PCM16 straight out of the
/// websocket's byte[] with no intermediate allocation.
public void WritePcm16(byte[] src, int offset, int byteCount)
{
int samples = byteCount >> 1;
if (samples <= 0) return;
long w = _write; // safe unsynchronised read: this thread is the only writer
for (int i = 0; i < samples; i++)
{
int b = offset + (i << 1);
short s = (short)(src[b] | (src[b + 1] << 8));
_buf[(int)((w + i) & _mask)] = s * (1f / 32768f);
}
Volatile.Write(ref _write, w + samples); // publish only once the data is really there
}
/// Producer thread only. Used to reproduce an exact gap the sender told us about, so a
/// dropped frame costs its own duration rather than shifting everything after it.
public void WriteSilence(int samples)
{
if (samples <= 0) return;
if (samples > _buf.Length) samples = _buf.Length;
long w = _write;
for (int i = 0; i < samples; i++) _buf[(int)((w + i) & _mask)] = 0f;
Volatile.Write(ref _write, w + samples);
}
/// Consumer read. The caller is responsible for staying within
/// <see cref="Capacity"/> of <see cref="WritePosition"/>.
public float this[long index] => _buf[(int)(index & _mask)];
}

286
tools/stub_relay.py Normal file
View File

@@ -0,0 +1,286 @@
"""Development stub relay for BingBong Live Voice.
Speaks the exact wire format the real relay (an endpoint on the AudioBank FastAPI service)
will use, so the mod can be built and tested before that backend exists. Deliberately minimal:
no auth, no session directory, no persistence. It is a test fixture, not a reference
implementation — build the real thing from the plan, not from this.
Requires `websockets` (pip install websockets).
python tools/stub_relay.py serve # relay on ws://localhost:8787/voice/ws
python tools/stub_relay.py tone # stream a 1 kHz sine into channel "test"
python tools/stub_relay.py tone --sweep # sweep 200-2000 Hz (resampling bugs are obvious)
python tools/stub_relay.py listen # dump received frames, to verify fan-out
Point the mod at it with, in BepInEx config:
[BingBong Voice]
RelayURL = ws://localhost:8787/voice/ws?token=listen:test
Leaving RelayURL empty instead derives the URL from BugleSoundAPIURL, which is where the
relay lives in production.
Connect as: ws://localhost:8787/voice/ws?token=listen:test
ws://localhost:8787/voice/ws?token=speak:test
Wire format — 12-byte little-endian header, then PCM16LE mono payload:
0 u8 magic 0xB1
1 u8 version 1
2 u8 codec 0=PCM16LE 1=mu-law
3 u8 rate code 0=16k 1=8k 2=24k 3=48k
4 u32 seq
8 u32 sampleTimestamp (samples since stream start)
Binary frame = audio. Text frame = JSON control.
"""
import argparse
import asyncio
import json
import math
import struct
import sys
from collections import defaultdict
from urllib.parse import urlparse, parse_qs
from websockets.asyncio.server import serve
from websockets.asyncio.client import connect
HOST = "localhost"
PORT = 8787
PATH = "/voice/ws"
MAGIC = 0xB1
VERSION = 1
CODEC_PCM16 = 0
RATE_CODES = {16000: 0, 8000: 1, 24000: 2, 48000: 3}
FRAME_MS = 20
HEADER = struct.Struct("<BBBBII")
def pack_header(seq: int, ts: int, rate: int = 16000, codec: int = CODEC_PCM16) -> bytes:
return HEADER.pack(MAGIC, VERSION, codec, RATE_CODES[rate], seq & 0xFFFFFFFF, ts & 0xFFFFFFFF)
# ---------------------------------------------------------------- relay
channels: dict[str, dict] = defaultdict(lambda: {"speaker": None, "listeners": set()})
def parse_token(raw_path: str):
"""token=<role>:<channel>. Returns (role, channel) or (None, None)."""
q = parse_qs(urlparse(raw_path).query)
token = (q.get("token") or [""])[0]
if ":" not in token:
return None, None
role, _, channel = token.partition(":")
if role not in ("listen", "speak") or not channel:
return None, None
return role, channel
async def handler(ws):
role, channel = parse_token(ws.request.path)
if role is None:
await ws.close(1008, "bad token")
return
ch = channels[channel]
if role == "speak":
old = ch["speaker"]
if old is not None:
try:
await old.send(json.dumps({"type": "preempted"}))
await old.close(1000, "preempted")
except Exception:
pass
ch["speaker"] = ws
print(f"[relay] speaker joined '{channel}' ({len(ch['listeners'])} listening)")
await broadcast_control(channel, {"type": "speaker_joined"})
try:
async for msg in ws:
if isinstance(msg, bytes):
await fanout(channel, msg)
except Exception:
pass
finally:
if ch["speaker"] is ws:
ch["speaker"] = None
print(f"[relay] speaker left '{channel}'")
await broadcast_control(channel, {"type": "speaker_left"})
return
ch["listeners"].add(ws)
print(f"[relay] listener joined '{channel}' -> {len(ch['listeners'])} listening")
try:
await ws.wait_closed()
finally:
ch["listeners"].discard(ws)
print(f"[relay] listener left '{channel}' -> {len(ch['listeners'])} listening")
async def fanout(channel: str, payload: bytes):
listeners = list(channels[channel]["listeners"])
if not listeners:
return
# Real relay uses a bounded per-listener queue and drops oldest; the stub just fires and
# forgets, which is fine locally and keeps this file short.
await asyncio.gather(*(safe_send(w, payload) for w in listeners), return_exceptions=True)
async def broadcast_control(channel: str, obj: dict):
msg = json.dumps(obj)
await asyncio.gather(
*(safe_send(w, msg) for w in list(channels[channel]["listeners"])),
return_exceptions=True,
)
async def safe_send(ws, payload):
try:
await ws.send(payload)
except Exception:
pass
async def cmd_serve(args):
async with serve(handler, HOST, args.port, max_size=None):
print(f"[relay] listening on ws://{HOST}:{args.port}{PATH}")
print(f"[relay] listen: ws://{HOST}:{args.port}{PATH}?token=listen:test")
print(f"[relay] speak : ws://{HOST}:{args.port}{PATH}?token=speak:test")
await asyncio.Future()
# ---------------------------------------------------------------- tone generator
try:
import numpy as _np
except ImportError:
_np = None
def generate_frame(phase, ts, n, rate, args):
"""Returns (new_phase, pcm16_bytes) for one frame, phase-continuous across frames."""
if _np is not None:
idx = _np.arange(n, dtype=_np.float64)
if args.sweep:
# 200-2000 Hz over 4 s: any resampling error shows up as an obvious warble
t = (ts + idx) / rate
freq = 200.0 + 1800.0 * (0.5 - 0.5 * _np.cos(2 * _np.pi * t / 4.0))
else:
freq = _np.full(n, float(args.freq))
ph = phase + _np.cumsum(2 * _np.pi * freq / rate)
samples = (_np.sin(ph) * 0.35 * 32767).astype("<i2")
return float(ph[-1] % (2 * _np.pi)), samples.tobytes()
# Fallback without numpy. Fine for a steady tone; sweep may not hold real time.
out = []
for i in range(n):
freq = args.freq
if args.sweep:
t = (ts + i) / rate
freq = 200.0 + 1800.0 * (0.5 - 0.5 * math.cos(2 * math.pi * t / 4.0))
phase = (phase + 2 * math.pi * freq / rate) % (2 * math.pi)
out.append(int(math.sin(phase) * 0.35 * 32767))
return phase, struct.pack(f"<{n}h", *out)
async def cmd_tone(args):
url = f"ws://{HOST}:{args.port}{PATH}?token=speak:{args.channel}"
rate = args.rate
n = int(rate * FRAME_MS / 1000)
print(f"[tone] {url} {rate} Hz, {n} samples/frame ({FRAME_MS} ms)")
loop = asyncio.get_running_loop()
async with connect(url, max_size=None) as ws:
seq = 0
ts = 0
phase = 0.0
# Deadline-based pacing. A plain `sleep(FRAME_MS)` per iteration sleeps for *at least*
# that long and then adds the generate+send time on top, so the sender silently drifts
# behind real time and starves the receiver's jitter buffer. Track an absolute deadline
# and sleep only for the remainder instead.
next_deadline = loop.time()
late_frames = 0
while True:
# Vectorised: generating 320 samples a frame in a Python loop costs more than the
# 20 ms frame budget once sweep mode adds a second transcendental per sample, and the
# sender then silently falls behind real time and starves the receiver.
phase, payload = generate_frame(phase, ts, n, rate, args)
await ws.send(pack_header(seq, ts, rate) + payload)
seq += 1
ts += n
next_deadline += FRAME_MS / 1000
delay = next_deadline - loop.time()
if delay > 0:
await asyncio.sleep(delay)
else:
# Generation itself overran the frame budget; we can't catch up by sleeping.
late_frames += 1
next_deadline = loop.time()
if seq % 50 == 0:
print(f"\r[tone] sent {seq} frames ({ts / rate:6.1f}s), {late_frames} late", end="", flush=True)
# ---------------------------------------------------------------- listener (verification)
async def cmd_listen(args):
url = f"ws://{HOST}:{args.port}{PATH}?token=listen:{args.channel}"
print(f"[listen] {url}")
async with connect(url, max_size=None) as ws:
count = 0
last_seq = None
gaps = 0
async for msg in ws:
if isinstance(msg, str):
print(f"\n[listen] control: {msg}")
continue
magic, ver, codec, rc, seq, ts = HEADER.unpack_from(msg, 0)
if magic != MAGIC or ver != VERSION:
print(f"\n[listen] BAD HEADER magic={magic:#x} ver={ver}")
continue
if last_seq is not None and seq != last_seq + 1:
gaps += 1
print(f"\n[listen] SEQ GAP {last_seq} -> {seq}")
last_seq = seq
count += 1
if count % 50 == 0:
print(
f"\r[listen] {count} frames, {len(msg) - 12} B payload, codec={codec} "
f"rate={rc} ts={ts} gaps={gaps}",
end="",
flush=True,
)
def main():
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("--port", type=int, default=PORT)
sub = p.add_subparsers(dest="cmd", required=True)
sub.add_parser("serve")
t = sub.add_parser("tone")
t.add_argument("--channel", default="test")
t.add_argument("--freq", type=float, default=1000.0)
t.add_argument("--rate", type=int, default=16000, choices=sorted(RATE_CODES))
t.add_argument("--sweep", action="store_true")
l = sub.add_parser("listen")
l.add_argument("--channel", default="test")
args = p.parse_args()
fn = {"serve": cmd_serve, "tone": cmd_tone, "listen": cmd_listen}[args.cmd]
try:
asyncio.run(fn(args))
except KeyboardInterrupt:
print("\nbye")
if __name__ == "__main__":
sys.exit(main())