Let all enabled Faces speak spatially
This commit is contained in:
parent
e75997cede
commit
129a2ce195
10 changed files with 128 additions and 19 deletions
|
|
@ -8,7 +8,7 @@ public sealed class AquariumAudioDocument
|
|||
|
||||
public static AquariumAudioDocument Empty { get; } = new();
|
||||
|
||||
public void EnqueuePcm16Base64(string base64Data, int sampleRate, int channels, float gain = 1.0f)
|
||||
public void EnqueuePcm16Base64(string base64Data, int sampleRate, int channels, float gain = 1.0f, float pan = 0.0f)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(base64Data) || sampleRate <= 0 || channels <= 0)
|
||||
{
|
||||
|
|
@ -47,7 +47,10 @@ public sealed class AquariumAudioDocument
|
|||
mono[frame] = Math.Clamp(sum / channels * safeGain, -1.0f, 1.0f);
|
||||
}
|
||||
|
||||
pcmChunks.Enqueue(new AquariumPcmAudioChunk(mono, sampleRate));
|
||||
var safePan = Math.Clamp(pan, -1.0f, 1.0f);
|
||||
var leftGain = safeGain * MathF.Sqrt((1.0f - safePan) * 0.5f);
|
||||
var rightGain = safeGain * MathF.Sqrt((1.0f + safePan) * 0.5f);
|
||||
pcmChunks.Enqueue(new AquariumPcmAudioChunk(mono, sampleRate, leftGain, rightGain));
|
||||
}
|
||||
|
||||
public IReadOnlyList<AquariumPcmAudioChunk> DrainPcmChunks(int maxChunks = 64)
|
||||
|
|
@ -62,4 +65,4 @@ public sealed class AquariumAudioDocument
|
|||
}
|
||||
}
|
||||
|
||||
public sealed record AquariumPcmAudioChunk(float[] MonoSamples, int SampleRate);
|
||||
public sealed record AquariumPcmAudioChunk(float[] MonoSamples, int SampleRate, float LeftGain = 1.0f, float RightGain = 1.0f);
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ internal sealed class AquariumSynthHost : IDisposable
|
|||
timeSeconds += Math.Max(deltaSeconds, 0.0f);
|
||||
foreach (var chunk in audio.DrainPcmChunks())
|
||||
{
|
||||
audioDevice.Play(chunk.MonoSamples, chunk.SampleRate);
|
||||
audioDevice.Play(chunk.MonoSamples, chunk.SampleRate, chunk.LeftGain, chunk.RightGain);
|
||||
}
|
||||
|
||||
if (!synth.Enabled)
|
||||
|
|
|
|||
|
|
@ -22,6 +22,11 @@ internal sealed class WasapiAudioDevice : IDisposable
|
|||
}
|
||||
|
||||
public void Play(ReadOnlySpan<float> monoSamples, int sourceSampleRate)
|
||||
{
|
||||
Play(monoSamples, sourceSampleRate, 1.0f, 1.0f);
|
||||
}
|
||||
|
||||
public void Play(ReadOnlySpan<float> monoSamples, int sourceSampleRate, float leftGain, float rightGain)
|
||||
{
|
||||
if (monoSamples.Length == 0)
|
||||
{
|
||||
|
|
@ -33,7 +38,7 @@ internal sealed class WasapiAudioDevice : IDisposable
|
|||
: Resample(monoSamples, sourceSampleRate, outputSampleRate);
|
||||
lock (sync)
|
||||
{
|
||||
voices.Add(new Voice(copy));
|
||||
voices.Add(new Voice(copy, Math.Clamp(leftGain, 0.0f, 4.0f), Math.Clamp(rightGain, 0.0f, 4.0f)));
|
||||
if (voices.Count > 64)
|
||||
{
|
||||
voices.RemoveRange(0, voices.Count - 64);
|
||||
|
|
@ -167,7 +172,7 @@ internal sealed class WasapiAudioDevice : IDisposable
|
|||
var sample = NextSample();
|
||||
for (var channel = 0; channel < channels; channel++)
|
||||
{
|
||||
*output++ = sample;
|
||||
*output++ = SampleForChannel(sample, channel, channels);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -177,10 +182,10 @@ internal sealed class WasapiAudioDevice : IDisposable
|
|||
var output = (short*)buffer;
|
||||
for (var frame = 0; frame < frameCount; frame++)
|
||||
{
|
||||
var sample = (short)Math.Clamp(NextSample() * short.MaxValue, short.MinValue, short.MaxValue);
|
||||
var sample = NextSample();
|
||||
for (var channel = 0; channel < channels; channel++)
|
||||
{
|
||||
*output++ = sample;
|
||||
*output++ = (short)Math.Clamp(SampleForChannel(sample, channel, channels) * short.MaxValue, short.MinValue, short.MaxValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -191,9 +196,25 @@ internal sealed class WasapiAudioDevice : IDisposable
|
|||
new Span<byte>((void*)buffer, bytes).Clear();
|
||||
}
|
||||
|
||||
private float NextSample()
|
||||
private static float SampleForChannel(StereoSample sample, int channel, int channelCount)
|
||||
{
|
||||
var sample = 0.0f;
|
||||
if (channelCount == 1)
|
||||
{
|
||||
return (sample.Left + sample.Right) * 0.5f;
|
||||
}
|
||||
|
||||
return channel switch
|
||||
{
|
||||
0 => sample.Left,
|
||||
1 => sample.Right,
|
||||
_ => (sample.Left + sample.Right) * 0.5f
|
||||
};
|
||||
}
|
||||
|
||||
private StereoSample NextSample()
|
||||
{
|
||||
var left = 0.0f;
|
||||
var right = 0.0f;
|
||||
lock (sync)
|
||||
{
|
||||
for (var index = voices.Count - 1; index >= 0; index--)
|
||||
|
|
@ -205,11 +226,13 @@ internal sealed class WasapiAudioDevice : IDisposable
|
|||
continue;
|
||||
}
|
||||
|
||||
sample += voice.Samples[voice.Position++];
|
||||
var sample = voice.Samples[voice.Position++];
|
||||
left += sample * voice.LeftGain;
|
||||
right += sample * voice.RightGain;
|
||||
}
|
||||
}
|
||||
|
||||
return Math.Clamp(sample, -1.0f, 1.0f);
|
||||
return new StereoSample(Math.Clamp(left, -1.0f, 1.0f), Math.Clamp(right, -1.0f, 1.0f));
|
||||
}
|
||||
|
||||
private static float[] Resample(ReadOnlySpan<float> source, int sourceSampleRate, int targetSampleRate)
|
||||
|
|
@ -234,9 +257,13 @@ internal sealed class WasapiAudioDevice : IDisposable
|
|||
return target;
|
||||
}
|
||||
|
||||
private sealed class Voice(float[] samples)
|
||||
private readonly record struct StereoSample(float Left, float Right);
|
||||
|
||||
private sealed class Voice(float[] samples, float leftGain, float rightGain)
|
||||
{
|
||||
public readonly float[] Samples = samples;
|
||||
public readonly float LeftGain = leftGain;
|
||||
public readonly float RightGain = rightGain;
|
||||
public int Position;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -57,9 +57,11 @@ public sealed class AquariumRuntime : IAquariumRuntime
|
|||
.Section("Face Voice")
|
||||
.Text("App Server", () => faceVoice.AppServerUri, value => faceVoice.AppServerUri = value.Trim(), "Loopback Codex app-server websocket, e.g. ws://127.0.0.1:8765.")
|
||||
.Toggle("Auto Route", () => faceVoice.AutoSelect, value => faceVoice.AutoSelect = value, "Routes speech to the nearest enabled Face endpoint by cursor/listener proximity.")
|
||||
.Slider("Audible Radius", () => faceVoice.AudibleRadius, value => faceVoice.AudibleRadius = value, 1.0f, 64.0f, "0.0", "Distance scale for Face voice gain and stereo pan.")
|
||||
.Readout("Active", () => faceVoice.ActiveSummary, "Currently selected Face endpoint and listener distance.")
|
||||
.Readout("Routes", () => faceVoice.RouteSummary, "Distances from the listener cursor to each Face endpoint.")
|
||||
.Text("Name", () => faceVoice.ActiveEndpoint.DisplayName, value => faceVoice.ActiveEndpoint.DisplayName = value.Trim(), "Display name for the active Face endpoint.")
|
||||
.Toggle("Enabled", () => faceVoice.ActiveEndpoint.Enabled, value => faceVoice.ActiveEndpoint.Enabled = value, "Allows this Face endpoint to speak and participate in auto routing.")
|
||||
.Text("Thread", () => faceVoice.ActiveEndpoint.ThreadId, value => faceVoice.ActiveEndpoint.ThreadId = value.Trim(), "Thread id that owns the active realtime Face session.")
|
||||
.Text("Voice", () => faceVoice.ActiveEndpoint.Voice, value => faceVoice.ActiveEndpoint.Voice = value.Trim(), "Realtime voice name, such as marin, cedar, or cove.")
|
||||
.Slider("Anchor X", () => faceVoice.ActiveEndpoint.Anchor.X, SetActiveFaceAnchorX, -32.0f, 32.0f, "0.0", "World X anchor for proximity routing.")
|
||||
|
|
@ -68,6 +70,7 @@ public sealed class AquariumRuntime : IAquariumRuntime
|
|||
.TextBox("Prompt", () => faceVoice.ActiveEndpoint.Prompt, value => faceVoice.ActiveEndpoint.Prompt = value, lines: 3, acceptsReturn: true, monospace: false, tooltip: "Backend realtime instructions for the active Face. This is speech behavior, not memory authority.")
|
||||
.TextBox("Say", () => faceVoiceText, value => faceVoiceText = value, lines: 2, acceptsReturn: false, submit: SendFaceVoiceText, monospace: false, tooltip: "Send one text utterance through the active Face route.")
|
||||
.Button("Start Active", faceVoice.StartActive, "Starts the active thread-scoped realtime session with audio output.")
|
||||
.Button("Start Enabled", faceVoice.StartAll, "Starts every enabled Face endpoint so rumination loops can speak regardless of user proximity.")
|
||||
.Button("Stop Active", faceVoice.StopActive, "Stops the active realtime Face voice session.")
|
||||
.Button("Clear Voice Log", faceVoice.ClearActiveLog, "Clears local transcript and audio counters for the active Face.")
|
||||
.Readout("Error", () => faceVoice.LastError, "Last realtime transport error for the active Face.")
|
||||
|
|
@ -111,6 +114,12 @@ public sealed class AquariumRuntime : IAquariumRuntime
|
|||
return $"Face voice stopping: {faceVoice.ActiveSummary}";
|
||||
}
|
||||
|
||||
if (verb == "startall")
|
||||
{
|
||||
faceVoice.StartAll();
|
||||
return "Enabled Face voice sessions starting";
|
||||
}
|
||||
|
||||
if (verb == "stopall")
|
||||
{
|
||||
faceVoice.StopAll();
|
||||
|
|
@ -134,6 +143,12 @@ public sealed class AquariumRuntime : IAquariumRuntime
|
|||
return $"Face voice auto route {faceVoice.AutoSelect}";
|
||||
}
|
||||
|
||||
if (verb == "radius" && args.Count > 1 && float.TryParse(args[1], NumberStyles.Float, CultureInfo.InvariantCulture, out var radius))
|
||||
{
|
||||
faceVoice.AudibleRadius = radius;
|
||||
return $"Face voice audible radius {faceVoice.AudibleRadius:0.0}";
|
||||
}
|
||||
|
||||
if (verb == "select" && args.Count > 1)
|
||||
{
|
||||
return faceVoice.Select(args[1])
|
||||
|
|
@ -167,7 +182,7 @@ public sealed class AquariumRuntime : IAquariumRuntime
|
|||
: $"Face voice endpoint {args[1]} was not removed";
|
||||
}
|
||||
|
||||
return "facevoice start | stop | stopall | say <text> | list | auto <true|false> | select <id> | add <id> <thread> <x> <y> [voice] | move <x> <y> | movehere | remove <id>";
|
||||
return "facevoice start | startall | stop | stopall | say <text> | list | auto <true|false> | radius <meters> | select <id> | add <id> <thread> <x> <y> [voice] | move <x> <y> | movehere | remove <id>";
|
||||
}, "Controls the Face realtime voice session.");
|
||||
}
|
||||
|
||||
|
|
@ -272,7 +287,8 @@ public sealed class AquariumRuntime : IAquariumRuntime
|
|||
FaceVoiceVoice = faceVoice.ActiveEndpoint.Voice,
|
||||
FaceVoicePrompt = faceVoice.ActiveEndpoint.Prompt,
|
||||
FaceVoiceEndpoints = faceVoice.ExportState(),
|
||||
FaceVoiceAutoSelect = faceVoice.AutoSelect
|
||||
FaceVoiceAutoSelect = faceVoice.AutoSelect,
|
||||
FaceVoiceAudibleRadius = faceVoice.AudibleRadius
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -52,6 +52,9 @@ public sealed class AquariumLiveState
|
|||
|
||||
[Key(13)]
|
||||
public bool FaceVoiceAutoSelect { get; set; } = true;
|
||||
|
||||
[Key(14)]
|
||||
public float FaceVoiceAudibleRadius { get; set; } = 24.0f;
|
||||
}
|
||||
|
||||
[MessagePackObject]
|
||||
|
|
|
|||
|
|
@ -37,6 +37,10 @@ public sealed class FaceVoiceEndpoint : IDisposable
|
|||
|
||||
public RealtimeFaceVoiceSession Session => session;
|
||||
|
||||
public float SpatialGain { get; private set; } = 1.0f;
|
||||
|
||||
public float SpatialPan { get; private set; }
|
||||
|
||||
public string Status => session.Status;
|
||||
|
||||
public string AudioStats => session.AudioStats;
|
||||
|
|
@ -53,6 +57,17 @@ public sealed class FaceVoiceEndpoint : IDisposable
|
|||
session.Prompt = Prompt;
|
||||
}
|
||||
|
||||
public void UpdateSpatial(Vector2 listenerWorld, float audibleRadius)
|
||||
{
|
||||
var delta = Anchor - listenerWorld;
|
||||
var distance = delta.Length();
|
||||
var safeRadius = Math.Max(audibleRadius, 0.001f);
|
||||
var normalizedDistance = Math.Clamp(distance / safeRadius, 0.0f, 1.0f);
|
||||
SpatialGain = Math.Clamp(1.0f / (1.0f + normalizedDistance * normalizedDistance * 3.0f), 0.18f, 1.0f);
|
||||
SpatialPan = Math.Clamp(delta.X / safeRadius, -0.95f, 0.95f);
|
||||
session.SetOutputSpatial(SpatialGain, SpatialPan);
|
||||
}
|
||||
|
||||
public AquariumFaceVoiceEndpointState ToState()
|
||||
{
|
||||
return new AquariumFaceVoiceEndpointState
|
||||
|
|
|
|||
|
|
@ -9,12 +9,14 @@ public sealed class FaceVoiceRouter : IDisposable
|
|||
private readonly AquariumAudioDocument audio;
|
||||
private readonly List<FaceVoiceEndpoint> endpoints = [];
|
||||
private int activeIndex;
|
||||
private float audibleRadius = 24.0f;
|
||||
|
||||
public FaceVoiceRouter(AquariumAudioDocument audio, AquariumLiveState state)
|
||||
{
|
||||
this.audio = audio;
|
||||
AppServerUri = state.FaceVoiceAppServerUri;
|
||||
AutoSelect = state.FaceVoiceAutoSelect;
|
||||
AudibleRadius = state.FaceVoiceAudibleRadius;
|
||||
|
||||
var endpointStates = state.FaceVoiceEndpoints.Count > 0
|
||||
? state.FaceVoiceEndpoints
|
||||
|
|
@ -34,6 +36,12 @@ public sealed class FaceVoiceRouter : IDisposable
|
|||
|
||||
public bool AutoSelect { get; set; } = true;
|
||||
|
||||
public float AudibleRadius
|
||||
{
|
||||
get => audibleRadius;
|
||||
set => audibleRadius = Math.Clamp(value, 1.0f, 128.0f);
|
||||
}
|
||||
|
||||
public Vector2 ListenerWorld { get; private set; }
|
||||
|
||||
public IReadOnlyList<FaceVoiceEndpoint> Endpoints => endpoints;
|
||||
|
|
@ -46,7 +54,7 @@ public sealed class FaceVoiceRouter : IDisposable
|
|||
{
|
||||
var endpoint = ActiveEndpoint;
|
||||
var distance = Vector2.Distance(ListenerWorld, endpoint.Anchor);
|
||||
return $"{endpoint.DisplayName} ({endpoint.Id}) {distance:0.0}m";
|
||||
return $"{endpoint.DisplayName} ({endpoint.Id}) {distance:0.0}m gain {endpoint.SpatialGain:0.00} pan {endpoint.SpatialPan:0.00}";
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -62,12 +70,17 @@ public sealed class FaceVoiceRouter : IDisposable
|
|||
{
|
||||
var mark = ReferenceEquals(endpoint, ActiveEndpoint) ? "*" : "";
|
||||
var distance = Vector2.Distance(ListenerWorld, endpoint.Anchor);
|
||||
return $"{mark}{endpoint.Id}:{distance:0.0}";
|
||||
return $"{mark}{endpoint.Id}:{distance:0.0} g{endpoint.SpatialGain:0.00} p{endpoint.SpatialPan:0.00}";
|
||||
}));
|
||||
|
||||
public void UpdateListener(Vector2 listenerWorld)
|
||||
{
|
||||
ListenerWorld = listenerWorld;
|
||||
foreach (var endpoint in endpoints)
|
||||
{
|
||||
endpoint.UpdateSpatial(listenerWorld, AudibleRadius);
|
||||
}
|
||||
|
||||
if (!AutoSelect || endpoints.Count <= 1)
|
||||
{
|
||||
return;
|
||||
|
|
@ -109,6 +122,7 @@ public sealed class FaceVoiceRouter : IDisposable
|
|||
},
|
||||
AppServerUri);
|
||||
endpoints.Add(endpoint);
|
||||
endpoint.UpdateSpatial(ListenerWorld, AudibleRadius);
|
||||
activeIndex = endpoints.Count - 1;
|
||||
AutoSelect = false;
|
||||
return endpoint;
|
||||
|
|
@ -122,6 +136,7 @@ public sealed class FaceVoiceRouter : IDisposable
|
|||
}
|
||||
|
||||
endpoint.ApplySessionSettings(AppServerUri);
|
||||
endpoint.UpdateSpatial(ListenerWorld, AudibleRadius);
|
||||
activeIndex = endpoints.IndexOf(endpoint);
|
||||
AutoSelect = false;
|
||||
return endpoint;
|
||||
|
|
@ -153,6 +168,16 @@ public sealed class FaceVoiceRouter : IDisposable
|
|||
ActiveEndpoint.Session.Start();
|
||||
}
|
||||
|
||||
public void StartAll()
|
||||
{
|
||||
foreach (var endpoint in endpoints.Where(endpoint => endpoint.Enabled))
|
||||
{
|
||||
endpoint.ApplySessionSettings(AppServerUri);
|
||||
endpoint.UpdateSpatial(ListenerWorld, AudibleRadius);
|
||||
endpoint.Session.Start();
|
||||
}
|
||||
}
|
||||
|
||||
public void StopActive()
|
||||
{
|
||||
ActiveEndpoint.Session.Stop();
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@ public sealed class RealtimeFaceVoiceSession : IDisposable
|
|||
private string remoteSdp = "";
|
||||
private int outputAudioChunks;
|
||||
private int outputAudioBytes;
|
||||
private float outputGain = 0.85f;
|
||||
private float outputPan;
|
||||
|
||||
public RealtimeFaceVoiceSession(AquariumAudioDocument audio)
|
||||
{
|
||||
|
|
@ -101,6 +103,15 @@ public sealed class RealtimeFaceVoiceSession : IDisposable
|
|||
}
|
||||
}
|
||||
|
||||
public void SetOutputSpatial(float gain, float pan)
|
||||
{
|
||||
lock (sync)
|
||||
{
|
||||
outputGain = Math.Clamp(gain, 0.0f, 1.5f);
|
||||
outputPan = Math.Clamp(pan, -1.0f, 1.0f);
|
||||
}
|
||||
}
|
||||
|
||||
public void Start()
|
||||
{
|
||||
lock (sync)
|
||||
|
|
@ -372,7 +383,15 @@ public sealed class RealtimeFaceVoiceSession : IDisposable
|
|||
var channels = audioObject?["numChannels"]?.GetValue<int?>() ?? 1;
|
||||
if (!string.IsNullOrWhiteSpace(data))
|
||||
{
|
||||
audio.EnqueuePcm16Base64(data, sampleRate, channels, 0.85f);
|
||||
float gain;
|
||||
float pan;
|
||||
lock (sync)
|
||||
{
|
||||
gain = outputGain;
|
||||
pan = outputPan;
|
||||
}
|
||||
|
||||
audio.EnqueuePcm16Base64(data, sampleRate, channels, gain, pan);
|
||||
}
|
||||
|
||||
lock (sync)
|
||||
|
|
|
|||
|
|
@ -140,3 +140,4 @@
|
|||
{"timestamp":"2026-05-18T08:03:00+01:00","topic":"zyphos-camera-composer-and-space-view","lesson":"Zyphos should not render the legacy height-field Grid in the space view. Aquarium frames now carry a 3D camera target and scene flags so clients can disable the height-field trace and request a starfield background. `ZyphosCameraComposer` separates tracked subject from parent composition anchor, borrowing the Cinemachine idea of composer-owned framing rather than embedding camera aim in world objects.","files":["src/Aquarium.Engine.Contracts/AquariumFrame.cs","src/Aquarium.Engine.Contracts/Render/AquariumRenderPlan.cs","src/Aquarium.Engine/Render/D3D12Renderer.cs","src/Aquarium.Engine/Render/Shaders/D3D12Scene.hlsl","src/Aquarium.Zyphos/ZyphosCameraComposer.cs","src/Aquarium.Zyphos/ZyphosRuntime.cs","src/Aquarium.Zyphos/ZyphosSceneBuilder.cs"],"source":"user saw the Zyphos screenshot, rejected the visible Grid, asked for starry background and camera locks informed by Cinemachine composer behavior"}
|
||||
{"timestamp":"2026-05-18T08:22:00+01:00","topic":"debug-ui-panel-scrolling","lesson":"DebugUi panels now own body scrolling, body clipping, and a visible scrollbar when controls exceed the viewport. Individual controls can still own local scrolling for text boxes and option popups, but panel overflow is an engine-level invariant so client panels cannot spill out of their frame.","files":["src/Aquarium.Engine/Render/Ui/DebugUi.cs"],"source":"user reported Zyphos UI did not fit its panel and expected all panels to have scrollbars"}
|
||||
{"timestamp":"2026-05-18T04:09:10+01:00","topic":"face-voice-proximity-routing","lesson":"Face voice is now routed through typed endpoint rows instead of one global session. `FaceVoiceRouter` selects the active endpoint by cursor/listener proximity when auto-route is enabled, while each `FaceVoiceEndpoint` owns its thread id, voice, prompt, enabled state, and XY anchor before delegating transport to `RealtimeFaceVoiceSession`. This keeps multi-Face speech routing explicit and spatial, while the next impurity to cut is manual XY placement: endpoint anchors should bind to real Face/body visual projections once those projections are exposed as a CPU-visible contract.","files":["src/Aquarium.Epiphany/Voice/FaceVoiceRouter.cs","src/Aquarium.Epiphany/Voice/FaceVoiceEndpoint.cs","src/Aquarium.Epiphany/AquariumRuntime.cs","src/Aquarium.Epiphany/State/AquariumLiveState.cs","state/map.yaml"],"source":"user asked to hook the voice path up while keeping in mind several Faces selected by spatial proximity"}
|
||||
{"timestamp":"2026-05-18T04:32:00+01:00","topic":"face-rumination-output-spatialization","lesson":"Face voice proximity is an addressing rule, not a speech-permission rule. Enabled Face endpoints can now run concurrently via `StartAll`, so active rumination loops may emit speech regardless of listener proximity; proximity updates only output gain and stereo pan. AquariumAudioDocument now carries per-chunk left/right gains and WasapiAudioDevice mixes stereo when the output device supports it, degrading to mono by averaging channels.","files":["src/Aquarium.Engine.Contracts/Audio/AquariumAudio.cs","src/Aquarium.Engine/Audio/WasapiAudioDevice.cs","src/Aquarium.Epiphany/Voice/RealtimeFaceVoiceSession.cs","src/Aquarium.Epiphany/Voice/FaceVoiceEndpoint.cs","src/Aquarium.Epiphany/Voice/FaceVoiceRouter.cs","src/Aquarium.Epiphany/AquariumRuntime.cs","state/map.yaml"],"source":"user clarified Faces should speak from active rumination loops regardless of proximity, using spatial audio only to indicate distance"}
|
||||
|
|
|
|||
|
|
@ -104,7 +104,7 @@ renderer_state:
|
|||
- Aquarium exposes an optional synth/audio hosting boundary through AquariumSynthDocument; Aquarium consumes AquaSynth.Faust as a pinned repo-local NuGet package so synth-agent construction cannot break Aquarium builds until the package is intentionally updated. AquaSynth.Core arrives transitively for patch editor/parser surfaces.
|
||||
- Aquarium now exposes a generic queued PCM audio lane through AquariumAudioDocument. Epiphany Face voice uses it to play streamed Codex realtime output without reaching into the engine's WASAPI internals.
|
||||
- Epiphany owns the first native Face realtime voice session controller. It stores loopback app-server URI, thread id, voice, and speech prompt in CultCache live state; starts `thread/realtime/start` with audio output over the experimental Codex app-server WebSocket JSON-RPC edge; sends text utterances; reflects ephemeral transcript/error/audio counters in the debug UI; and treats speech as transient Face surface, not durable memory or project authority.
|
||||
- Face voice routing is no longer a singleton. Epiphany persists typed FaceVoiceEndpoint rows with id, display name, thread id, voice, prompt, enabled flag, and XY proximity anchor; FaceVoiceRouter selects the active endpoint from cursor/listener proximity when auto-route is enabled, while terminal commands can add, select, move, remove, start, stop, and speak through endpoints.
|
||||
- Face voice routing is no longer a singleton. Epiphany persists typed FaceVoiceEndpoint rows with id, display name, thread id, voice, prompt, enabled flag, XY proximity anchor, and an audible-radius scale. FaceVoiceRouter uses proximity only to choose the addressed endpoint for user input; enabled endpoints can all run concurrently so active rumination loops may speak regardless of listener distance. Distance shapes PCM output gain and stereo pan through AquariumAudioDocument/WasapiAudioDevice.
|
||||
- Global GraphicsSettings document feeding render debug, exposure, bloom intensity, and bloom veil.
|
||||
- Temporal resolve uses scene color, metadata, and control history only; dormant event history lanes are cut until a real producer and consumer contract exists.
|
||||
- Isolated Epiphany agent preview renders run through the real D3D12 renderer from src/Aquarium.Epiphany.AgentPreview; preview isolation is owned by a preview-only scene shader, while the engine exposes only final-frame PNG capture.
|
||||
|
|
|
|||
Loading…
Reference in a new issue