Soul pass 3: conditional enable in Runtime, receipt floor, one write path
- The enable commit is conditional (ExpectUnchanged) and lives in the new Runtime BrokkrCommandStore, with the pull-recompute-retry loop in BrokkrCommandDrain.EnableAsync, so the tests exercise the shipped code. An intent landing between the pull and the commit is expired, not run; eight refusals in a row fail the enable loudly with the sink off. The probe is now a thin fault-injecting wrapper over the real store, not a copy of it. - Every editor write to the command lane is one cache commit; durable shard logs are off (no server, no reader). - No receipt is pruned before ten minutes, whatever the count, and the CLI refuses a wait over five minutes. Fixed the comments that claimed otherwise. - Pending and Prunable count a document only under the key its own id names. - README and snapshot comment: the token stops stale EditorPrefs, it does not authenticate writers; a restored or sync-conflict copy of the editor's own store can rerun answered intents. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
878d6a52b5
commit
121fa329e2
13 changed files with 338 additions and 148 deletions
|
|
@ -105,9 +105,15 @@ internal static class Cli
|
|||
|
||||
private static async Task<int> WriteIntentAsync(CultCache cache, string action, Dictionary<string, string> options)
|
||||
{
|
||||
var waitMilliseconds = options.TryGetValue("--wait-ms", out var wait) ? Integer("--wait-ms", wait) : 10000;
|
||||
// The editor keeps every receipt for at least BrokkrCommandDrain.ReceiptFloor, which is what makes a wait this
|
||||
// long safe: the receipt a caller waits on cannot be pruned before it reads it.
|
||||
if (waitMilliseconds > BrokkrCommandDrain.MaxCallerWait.TotalMilliseconds)
|
||||
throw new ArgumentException($"--wait-ms may not exceed {BrokkrCommandDrain.MaxCallerWait.TotalMilliseconds:F0} (the editor keeps receipts for at least {BrokkrCommandDrain.ReceiptFloor.TotalMinutes:F0} minutes).");
|
||||
// A command id is single-use, so every invocation is a new act with a new id. --command-id exists for callers
|
||||
// that need to name their own; one that already has a receipt is refused, because that receipt answered an
|
||||
// earlier act and would be reported as the answer to this one. The check and the write are two steps, so two\n // callers naming the same new id in the same instant can still collide; ids are minted, and that is a caller defect.
|
||||
// earlier act and would be reported as the answer to this one. The check and the write are two steps, so two
|
||||
// callers naming the same new id in the same instant can still collide; ids are minted, and that is a caller defect.
|
||||
var named = options.TryGetValue("--command-id", out var commandId) && !string.IsNullOrWhiteSpace(commandId);
|
||||
if (named)
|
||||
{
|
||||
|
|
@ -129,7 +135,7 @@ internal static class Cli
|
|||
await cache.AddAsync(command, new CultRecordHandle<BrokkrUnityCommand>(BrokkrCommandLedger.CommandKey(command.commandId)));
|
||||
await cache.FlushAsync();
|
||||
|
||||
var waitMilliseconds = options.TryGetValue("--wait-ms", out var wait) ? Integer("--wait-ms", wait) : 10000;
|
||||
|
||||
if (waitMilliseconds <= 0)
|
||||
{
|
||||
Console.WriteLine(command.commandId);
|
||||
|
|
|
|||
|
|
@ -317,4 +317,17 @@ public sealed class BrokkrCommandCliTests
|
|||
Assert.DoesNotContain("Timed out", result.Error);
|
||||
Assert.False(string.IsNullOrWhiteSpace(result.Error));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AWaitLongerThanTheReceiptFloorAllowsIsRefusedBeforeAnythingIsWritten()
|
||||
{
|
||||
using var project = new ScratchProject();
|
||||
var tooLong = (BrokkrCommandDrain.MaxCallerWait.TotalMilliseconds + 1).ToString("F0");
|
||||
|
||||
var result = await RunAsync("--unity-cache", project.StorePath, "--action", "refreshAssets", "--wait-ms", tooLong);
|
||||
|
||||
Assert.Equal(2, result.Exit);
|
||||
Assert.Contains("--wait-ms may not exceed", result.Error);
|
||||
Assert.Empty(StoredIntents(project));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -250,22 +250,34 @@ public sealed class BrokkrCommandDrainTests
|
|||
|
||||
await project.WriteCommandAsync("fresh");
|
||||
var slowest = TimeSpan.Zero;
|
||||
var ticks = 0;
|
||||
do
|
||||
for (var tick = 1; tick <= 4; tick++)
|
||||
{
|
||||
clock.Restart();
|
||||
await editor.TickAsync();
|
||||
slowest = clock.Elapsed > slowest ? clock.Elapsed : slowest;
|
||||
output.WriteLine($"tick {++ticks}: {clock.Elapsed.TotalMilliseconds:F0} ms, receipts kept {Receipts(project).Length}");
|
||||
if (ticks == 1)
|
||||
output.WriteLine($"tick {tick}: {clock.Elapsed.TotalMilliseconds:F0} ms");
|
||||
if (tick == 1)
|
||||
Assert.Equal(new[] { "fresh" }, editor.Executed.Select(command => command.commandId));
|
||||
}
|
||||
while (Receipts(project).Length > BrokkrCommandDrain.RetainedReceipts && ticks < 40);
|
||||
|
||||
// Every receipt is younger than the retention floor, so none is pruned however far over the count: a caller
|
||||
// waiting on an expired receipt still reads it.
|
||||
Assert.Equal(stale + 1, Receipts(project).Length);
|
||||
|
||||
editor.ClockSkew = BrokkrCommandDrain.ReceiptFloor + TimeSpan.FromMinutes(1);
|
||||
var ticks = 0;
|
||||
while (Receipts(project).Length > BrokkrCommandDrain.RetainedReceipts && ticks < 40)
|
||||
{
|
||||
clock.Restart();
|
||||
await editor.TickAsync();
|
||||
slowest = clock.Elapsed > slowest ? clock.Elapsed : slowest;
|
||||
output.WriteLine($"aged tick {++ticks}: {clock.Elapsed.TotalMilliseconds:F0} ms");
|
||||
}
|
||||
|
||||
Assert.True(slowest < TickBudget, $"slowest tick took {slowest}");
|
||||
Assert.Equal(new[] { "fresh" }, editor.Executed.Select(command => command.commandId));
|
||||
Assert.Equal("accepted", StatusOf(project, "fresh"));
|
||||
// Receipts are bounded, and an intent goes with its receipt: nothing is left to run again.
|
||||
// Receipts are bounded once they age, and an intent goes with its receipt: nothing is left to run again.
|
||||
using var audit = project.OpenCache();
|
||||
var documents = audit.AllStoredDocuments.Select(entry => entry.Document).ToArray();
|
||||
Assert.Equal(BrokkrCommandDrain.RetainedReceipts, documents.OfType<BrokkrUnityCommandReceipt>().Count());
|
||||
|
|
@ -273,6 +285,67 @@ public sealed class BrokkrCommandDrainTests
|
|||
Assert.False(new BrokkrCommandLedger(audit).TryNextPending(out _));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AnIntentLandingBetweenTheEnablesPullAndItsCommitIsExpiredNotRun()
|
||||
{
|
||||
using var project = new ScratchProject();
|
||||
using var editor = await EditorProbe.StartAsync(project.StorePath);
|
||||
editor.DisableSink();
|
||||
await project.WriteCommandAsync("waiting");
|
||||
editor.BeforeEnableCommit = attempt =>
|
||||
{
|
||||
if (attempt == 1)
|
||||
project.WriteCommandAsync("racer").GetAwaiter().GetResult();
|
||||
};
|
||||
|
||||
var expired = await editor.EnableSinkAsync();
|
||||
await editor.TickAsync();
|
||||
await editor.TickAsync();
|
||||
|
||||
Assert.Equal(2, editor.EnableAttempts);
|
||||
Assert.Equal(2, expired);
|
||||
Assert.Empty(editor.Executed);
|
||||
Assert.Equal("expired", StatusOf(project, "racer"));
|
||||
Assert.Equal("expired", StatusOf(project, "waiting"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AWriterThatNeverLetsTheStoreSettleMakesTheEnableFailAndLeavesTheSinkOff()
|
||||
{
|
||||
using var project = new ScratchProject();
|
||||
using var editor = await EditorProbe.StartAsync(project.StorePath);
|
||||
editor.DisableSink();
|
||||
editor.BeforeEnableCommit = attempt => project.WriteCommandAsync($"stream-{attempt}").GetAwaiter().GetResult();
|
||||
|
||||
var error = await Assert.ThrowsAsync<InvalidOperationException>(editor.EnableSinkAsync);
|
||||
|
||||
Assert.Contains("continuously", error.Message);
|
||||
Assert.Equal(BrokkrCommandDrain.MaxEnableAttempts, editor.EnableAttempts);
|
||||
Assert.False(editor.AgentCommandsEnabled);
|
||||
editor.BeforeEnableCommit = null;
|
||||
await editor.TickAsync();
|
||||
Assert.Empty(editor.Executed);
|
||||
Assert.Empty(Receipts(project));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AWriterThatStopsLetsTheEnableConvergeAndExpireEverythingItWrote()
|
||||
{
|
||||
using var project = new ScratchProject();
|
||||
using var editor = await EditorProbe.StartAsync(project.StorePath);
|
||||
editor.DisableSink();
|
||||
editor.BeforeEnableCommit = attempt =>
|
||||
{
|
||||
if (attempt <= 3)
|
||||
project.WriteCommandAsync($"stream-{attempt}").GetAwaiter().GetResult();
|
||||
};
|
||||
|
||||
Assert.Equal(3, await editor.EnableSinkAsync());
|
||||
await editor.TickAsync();
|
||||
|
||||
Assert.Equal(4, editor.EnableAttempts);
|
||||
Assert.Empty(editor.Executed);
|
||||
}
|
||||
[Fact]
|
||||
public async Task DisallowedActionIsDeniedAndNeverExecuted()
|
||||
{
|
||||
|
|
|
|||
|
|
@ -158,4 +158,28 @@ public sealed class BrokkrCommandLedgerTests
|
|||
using var audit = project.OpenCache();
|
||||
Assert.Equal(ids.Length, audit.AllStoredDocuments.Count(entry => entry.Document is BrokkrUnityCommandReceipt));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DocumentUnderAKeyItsIdDoesNotNameIsNeverPending()
|
||||
{
|
||||
// One body id under two keys must not run twice, and a receipt under the wrong key retires nothing.
|
||||
using var project = new ScratchProject();
|
||||
using (var writer = project.OpenCache())
|
||||
{
|
||||
var command = new BrokkrUnityCommand { commandId = "dup", action = "refreshAssets" };
|
||||
await writer.AddAsync(command, new CultRecordHandle<BrokkrUnityCommand>(BrokkrCommandLedger.CommandKey("dup")));
|
||||
await writer.AddAsync(new BrokkrUnityCommand { commandId = "dup", action = "refreshAssets" },
|
||||
new CultRecordHandle<BrokkrUnityCommand>(BrokkrCommandLedger.CommandKey("other")));
|
||||
await writer.AddAsync(new BrokkrUnityCommand { commandId = "victim", action = "refreshAssets" },
|
||||
new CultRecordHandle<BrokkrUnityCommand>(BrokkrCommandLedger.CommandKey("victim")));
|
||||
await writer.AddAsync(new BrokkrUnityCommandReceipt { commandId = "victim", status = "accepted" },
|
||||
new CultRecordHandle<BrokkrUnityCommandReceipt>(new CultRecordKey("unity/receipts/elsewhere")));
|
||||
await writer.FlushAsync();
|
||||
}
|
||||
|
||||
using var editor = project.OpenCache();
|
||||
var pending = new BrokkrCommandLedger(editor).Pending(int.MaxValue);
|
||||
|
||||
Assert.Equal(new[] { "dup", "victim" }, pending.Select(command => command.commandId).OrderBy(id => id));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,14 +7,16 @@ using GameCult.Networking;
|
|||
namespace Brokkr.Contracts.Tests;
|
||||
|
||||
// The scenario harness: an editor without Unity. It opens the store exactly as BrokkrCultMeshMirror does (a CultMesh
|
||||
// node with StartServer=false over a directory store and durable shard logs) and runs one service tick in the
|
||||
// node with StartServer=false over a directory store, no shard log) and runs one service tick in the
|
||||
// service order: pull, drain pending intents through BrokkrCommandDrain (the same class the service uses), host
|
||||
// snapshot, soft flush. "Executing" an intent only records it, so the scenarios measure delivery and once-only
|
||||
// execution, not Unity. It is also the drain's IBrokkrCommandStore, with knobs to make the store fail.
|
||||
// execution, not Unity. It is also a thin IBrokkrCommandStore in front of the real BrokkrCommandStore, with knobs to
|
||||
// make writes fail and to land a writer between the enable's pull and its commit.
|
||||
internal sealed class EditorProbe : IDisposable, IBrokkrCommandStore
|
||||
{
|
||||
private readonly CultMeshNode node;
|
||||
private readonly BrokkrCommandLedger ledger;
|
||||
private readonly BrokkrCommandStore store;
|
||||
private readonly BrokkrCommandPolicy policy = new("probe", BrokkrCommandPolicy.DefaultAllowedActions.Concat(new[]
|
||||
{
|
||||
"createPrefabVariant", "createScriptableObject"
|
||||
|
|
@ -38,6 +40,13 @@ internal sealed class EditorProbe : IDisposable, IBrokkrCommandStore
|
|||
internal Func<BrokkrUnityCommandReceipt, bool>? FailReceiptPut { get; set; }
|
||||
internal Func<bool>? FailFlush { get; set; }
|
||||
|
||||
// Runs after the enable has pulled and computed, just before its conditional commit; a writer landing an intent here is the race.
|
||||
internal Action<int>? BeforeEnableCommit { get; set; }
|
||||
internal int EnableAttempts { get; private set; }
|
||||
|
||||
// Added to the drain's clock, to age receipts past the retention floor.
|
||||
internal TimeSpan ClockSkew { get; set; }
|
||||
|
||||
// Flushes the drain asked for; each is an fsync in the real store.
|
||||
internal int Flushes { get; private set; }
|
||||
|
||||
|
|
@ -45,6 +54,7 @@ internal sealed class EditorProbe : IDisposable, IBrokkrCommandStore
|
|||
{
|
||||
this.node = node;
|
||||
ledger = new BrokkrCommandLedger(node.Cache);
|
||||
store = new BrokkrCommandStore(node.Cache);
|
||||
}
|
||||
|
||||
// enabledInStore false is an editor whose EditorPrefs still hold a token, opening a store that never saw that
|
||||
|
|
@ -56,7 +66,7 @@ internal sealed class EditorProbe : IDisposable, IBrokkrCommandStore
|
|||
{
|
||||
StartServer = false,
|
||||
CacheOptions = new CultCacheOpenOptions { UseDirectoryStore = true },
|
||||
EnableDurableShardLogs = true,
|
||||
EnableDurableShardLogs = false,
|
||||
DatabaseOptions = new CultNetDatabaseOptions { RuntimeId = "brokkr-unity-editor" }
|
||||
});
|
||||
var probe = new EditorProbe(node);
|
||||
|
|
@ -82,7 +92,8 @@ internal sealed class EditorProbe : IDisposable, IBrokkrCommandStore
|
|||
observedAt = DateTime.UtcNow.ToString("O")
|
||||
};
|
||||
},
|
||||
probe);
|
||||
probe,
|
||||
() => DateTimeOffset.UtcNow + probe.ClockSkew);
|
||||
|
||||
// The operator ticks the toggle off.
|
||||
internal void DisableSink() => SinkToken = null;
|
||||
|
|
@ -91,6 +102,7 @@ internal sealed class EditorProbe : IDisposable, IBrokkrCommandStore
|
|||
internal async Task<int> EnableSinkAsync()
|
||||
{
|
||||
await ledger.PullAsync();
|
||||
EnableAttempts = 0;
|
||||
var token = Guid.NewGuid().ToString("N");
|
||||
var expired = await NewDrain(this).EnableAsync(token);
|
||||
SinkToken = token;
|
||||
|
|
@ -124,47 +136,32 @@ internal sealed class EditorProbe : IDisposable, IBrokkrCommandStore
|
|||
return handled;
|
||||
}
|
||||
|
||||
// The real store with failure and interleaving knobs in front of it; no store logic lives here.
|
||||
public Task PutReceiptAsync(BrokkrUnityCommandReceipt receipt)
|
||||
{
|
||||
if (FailReceiptPut?.Invoke(receipt) == true)
|
||||
throw new IOException("receipt write failed");
|
||||
return node.Database.PutAsync(BrokkrCommandLedger.ReceiptKey(receipt.commandId), receipt);
|
||||
return store.PutReceiptAsync(receipt);
|
||||
}
|
||||
|
||||
public Task DeleteReceiptAsync(string commandId) =>
|
||||
node.Database.DeleteAsync<BrokkrUnityCommandReceipt>(BrokkrCommandLedger.ReceiptKey(commandId));
|
||||
public Task DeleteReceiptAsync(string commandId) => store.DeleteReceiptAsync(commandId);
|
||||
|
||||
public Task CommitEnableAsync(BrokkrUnityCommandReceipt[] expired, BrokkrSinkEnabled marker)
|
||||
public async Task<bool> TryCommitEnableAsync(BrokkrUnityCommandReceipt[] expired, BrokkrSinkEnabled marker)
|
||||
{
|
||||
Assert.True(node.Cache.Commit(batch =>
|
||||
{
|
||||
foreach (var receipt in expired)
|
||||
batch.Upsert(receipt, new CultRecordHandle<BrokkrUnityCommandReceipt>(BrokkrCommandLedger.ReceiptKey(receipt.commandId)));
|
||||
batch.Upsert(marker, new CultRecordHandle<BrokkrSinkEnabled>(BrokkrCommandLedger.SinkKey));
|
||||
}));
|
||||
return Task.CompletedTask;
|
||||
EnableAttempts++;
|
||||
BeforeEnableCommit?.Invoke(EnableAttempts);
|
||||
return await store.TryCommitEnableAsync(expired, marker);
|
||||
}
|
||||
|
||||
public Task DeleteAnsweredAsync(string[] commandIds)
|
||||
{
|
||||
Assert.True(node.Cache.Commit(batch =>
|
||||
{
|
||||
foreach (var commandId in commandIds)
|
||||
{
|
||||
batch.Remove(BrokkrCommandLedger.CommandKey(commandId));
|
||||
batch.Remove(BrokkrCommandLedger.ReceiptKey(commandId));
|
||||
}
|
||||
}));
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
public Task DeleteAnsweredAsync(string[] commandIds) => store.DeleteAnsweredAsync(commandIds);
|
||||
|
||||
public Task FlushAsync()
|
||||
{
|
||||
if (FailFlush?.Invoke() == true)
|
||||
throw new IOException("flush failed");
|
||||
Flushes++;
|
||||
return node.FlushAsync(soft: true);
|
||||
return store.FlushAsync();
|
||||
}
|
||||
|
||||
public void Dispose() => node.Dispose();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -12,9 +12,8 @@ using UnityEngine;
|
|||
namespace GameCult.Brokkr.Editor
|
||||
{
|
||||
// The one open handle on the project's directory store. BrokkrEditorService owns the only instance; the window
|
||||
// publishes through it and never opens the store itself. It is also the drain's IBrokkrCommandStore: puts and
|
||||
// deletes stage, and FlushAsync is the only fsync.
|
||||
internal sealed class BrokkrCultMeshMirror : IDisposable, IBrokkrCommandStore
|
||||
// publishes through it and never opens the store itself.
|
||||
internal sealed class BrokkrCultMeshMirror : IDisposable
|
||||
{
|
||||
private readonly Queue<BrokkrSyncReceipt> syncReceiptQueue = new();
|
||||
private CultMeshNode node;
|
||||
|
|
@ -24,6 +23,11 @@ namespace GameCult.Brokkr.Editor
|
|||
internal string CachePath { get; private set; } = "";
|
||||
internal BrokkrCommandLedger Ledger { get; private set; }
|
||||
|
||||
// Every write the editor makes to the command lane (receipts, the enable, the prune) is one commit through
|
||||
// this store, the same class the scenario harness runs. Snapshots, sync documents and the window's own
|
||||
// intents still go through Database.PutAsync.
|
||||
internal BrokkrCommandStore Store { get; private set; }
|
||||
|
||||
internal async Task StartAsync(string cachePath)
|
||||
{
|
||||
if (node != null)
|
||||
|
|
@ -43,13 +47,15 @@ namespace GameCult.Brokkr.Editor
|
|||
{
|
||||
UseDirectoryStore = true
|
||||
},
|
||||
EnableDurableShardLogs = true,
|
||||
// No server and no reader of the shard log, so it would only add a log write to every change.
|
||||
EnableDurableShardLogs = false,
|
||||
DatabaseOptions = new CultNetDatabaseOptions
|
||||
{
|
||||
RuntimeId = "brokkr-unity-editor"
|
||||
}
|
||||
});
|
||||
Ledger = new BrokkrCommandLedger(node.Cache);
|
||||
Store = new BrokkrCommandStore(node.Cache);
|
||||
|
||||
syncReceiptSubscription = node.Database
|
||||
.Watch<BrokkrSyncReceipt>()
|
||||
|
|
@ -75,60 +81,6 @@ namespace GameCult.Brokkr.Editor
|
|||
await node.FlushAsync(soft: true);
|
||||
}
|
||||
|
||||
// The receipt is the only proof an intent ran, and it is keyed by the intent it answers.
|
||||
public Task PutReceiptAsync(BrokkrUnityCommandReceipt receipt)
|
||||
{
|
||||
RequireRunning();
|
||||
if (string.IsNullOrWhiteSpace(receipt.commandId))
|
||||
{
|
||||
throw new ArgumentException("A receipt must name the command it answers.", nameof(receipt));
|
||||
}
|
||||
|
||||
return node.Database.PutAsync(BrokkrCommandLedger.ReceiptKey(receipt.commandId), receipt);
|
||||
}
|
||||
|
||||
public Task DeleteReceiptAsync(string commandId)
|
||||
{
|
||||
RequireRunning();
|
||||
return node.Database.DeleteAsync<BrokkrUnityCommandReceipt>(BrokkrCommandLedger.ReceiptKey(commandId));
|
||||
}
|
||||
|
||||
// The batch methods commit through the cache directly. Database.PutAsync appends a mutation-log entry per
|
||||
// record, which is what made 2000 expiries cost seconds; nothing here is replicated (no listener), and the
|
||||
// ledger and the CLI both read the cache and the store.
|
||||
public Task CommitEnableAsync(BrokkrUnityCommandReceipt[] expired, BrokkrSinkEnabled marker)
|
||||
{
|
||||
RequireRunning();
|
||||
var committed = node.Cache.Commit(batch =>
|
||||
{
|
||||
foreach (var receipt in expired)
|
||||
{
|
||||
batch.Upsert(receipt, new CultRecordHandle<BrokkrUnityCommandReceipt>(BrokkrCommandLedger.ReceiptKey(receipt.commandId)));
|
||||
}
|
||||
|
||||
batch.Upsert(marker, new CultRecordHandle<BrokkrSinkEnabled>(BrokkrCommandLedger.SinkKey));
|
||||
});
|
||||
return committed ? Task.CompletedTask : Task.FromException(new InvalidOperationException("The enable commit was refused."));
|
||||
}
|
||||
|
||||
public Task DeleteAnsweredAsync(string[] commandIds)
|
||||
{
|
||||
RequireRunning();
|
||||
var committed = node.Cache.Commit(batch =>
|
||||
{
|
||||
foreach (var commandId in commandIds)
|
||||
{
|
||||
batch.Remove(BrokkrCommandLedger.CommandKey(commandId));
|
||||
batch.Remove(BrokkrCommandLedger.ReceiptKey(commandId));
|
||||
}
|
||||
});
|
||||
return committed ? Task.CompletedTask : Task.FromException(new InvalidOperationException("The prune commit was refused."));
|
||||
}
|
||||
public Task FlushAsync()
|
||||
{
|
||||
RequireRunning();
|
||||
return node.FlushAsync(soft: true);
|
||||
}
|
||||
// Command ids are single-use: a caller with no id gets a fresh one, and reusing an answered id is a defect.
|
||||
internal async Task PublishCommandAsync(BrokkrUnityCommand command)
|
||||
{
|
||||
|
|
@ -210,6 +162,7 @@ namespace GameCult.Brokkr.Editor
|
|||
node?.Dispose();
|
||||
node = null;
|
||||
Ledger = null;
|
||||
Store = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -74,7 +74,6 @@ namespace GameCult.Brokkr.Editor
|
|||
throw new InvalidOperationException("The Brokkr mirror did not start; the sink stays off.");
|
||||
}
|
||||
|
||||
await MirrorInstance.PullExternalUpdatesAsync();
|
||||
var token = Guid.NewGuid().ToString("N");
|
||||
var expired = await NewDrain().EnableAsync(token);
|
||||
BrokkrSettings.EnableAgentCommands(token);
|
||||
|
|
@ -156,7 +155,7 @@ namespace GameCult.Brokkr.Editor
|
|||
MirrorInstance.Ledger,
|
||||
new BrokkrCommandPolicy(projectRoot, BrokkrSettings.AllowedAgentActions),
|
||||
BrokkrUnityCommandExecutor.Execute,
|
||||
MirrorInstance);
|
||||
MirrorInstance.Store);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,21 +4,20 @@ using System.Threading.Tasks;
|
|||
|
||||
namespace GameCult.Brokkr
|
||||
{
|
||||
// What the drain writes through. BrokkrCultMeshMirror is the editor's implementation and the scenario harness
|
||||
// has its own. Writes only stage and FlushAsync makes them durable (an fsync), so anything that must land
|
||||
// together goes through one of the two batch methods, each a single atomic commit: the cost of a write is per
|
||||
// commit, not per record.
|
||||
// What the drain writes through. BrokkrCommandStore is the real one; the scenario harness wraps it to inject
|
||||
// failures. Writes only stage and FlushAsync makes them durable (an fsync), so anything that must land together
|
||||
// goes through one of the batch methods, each a single atomic commit.
|
||||
public interface IBrokkrCommandStore
|
||||
{
|
||||
Task PutReceiptAsync(BrokkrUnityCommandReceipt receipt);
|
||||
Task DeleteReceiptAsync(string commandId);
|
||||
// Every expiry receipt and the enable marker, or none of them.
|
||||
Task CommitEnableAsync(BrokkrUnityCommandReceipt[] expired, BrokkrSinkEnabled marker);
|
||||
// Every expiry receipt and the enable marker, or none of them. False when the store changed since the
|
||||
// caller's pull, and nothing was written.
|
||||
Task<bool> TryCommitEnableAsync(BrokkrUnityCommandReceipt[] expired, BrokkrSinkEnabled marker);
|
||||
// Each intent with its receipt, or none of them.
|
||||
Task DeleteAnsweredAsync(string[] commandIds);
|
||||
Task FlushAsync();
|
||||
}
|
||||
|
||||
// The one decision path from a pending intent to a receipt. BrokkrEditorService supplies the executor and the
|
||||
// store; the scenario harness supplies fakes. Nothing else may run an intent.
|
||||
//
|
||||
|
|
@ -36,18 +35,30 @@ namespace GameCult.Brokkr
|
|||
// could not be written, and the next drain rewrites it as interrupted
|
||||
// interrupted an attempt that did not finish; it is never run again automatically, the operator decides
|
||||
// accepted / failed / ... the executor's verdict
|
||||
// A marker that could not be flushed is withdrawn, so an intent that never started is retried, not reported
|
||||
// as interrupted.
|
||||
// A marker that could not be written or flushed is withdrawn, so an intent that never started is normally
|
||||
// retried. If the store fails in a way that leaves the marker behind, the intent is reported as interrupted,
|
||||
// which errs safe.
|
||||
//
|
||||
// The token is not authentication. It stops a stale "enabled" in EditorPrefs from authorizing a store this
|
||||
// editor never enabled (a re-clone, a foreign copy). Anyone with write access to .brokkr can write intents, and
|
||||
// the Allowed Actions policy is what limits them; nothing here proves who wrote a document.
|
||||
public sealed class BrokkrCommandDrain
|
||||
{
|
||||
// The most intents one drain executes or denies. A flood cannot hold the editor's update loop.
|
||||
public const int MaxPerTick = 16;
|
||||
|
||||
// Answered intents keep their receipt (the CLI reads its own) until this many newer receipts exist; then the
|
||||
// oldest command and receipt are deleted together, at most MaxPrunedPerTick pairs per drain. A caller
|
||||
// waiting for a receipt would need this many newer answers inside its wait to lose it.
|
||||
// Receipts are kept for the newest RetainedReceipts answers, and never for less than ReceiptFloor: a
|
||||
// receipt younger than that is not pruned whatever the count, so a caller waiting on it (at most
|
||||
// MaxCallerWait, which brokkr-command enforces) always reads it. Older receipts beyond the count are
|
||||
// deleted with their intents, at most MaxPrunedPerTick pairs per drain. The floor is measured on the
|
||||
// editor's own clock against receipts the editor stamped. Id reuse is refused only while the receipt exists.
|
||||
public const int RetainedReceipts = 1000;
|
||||
public const int MaxPrunedPerTick = 128;
|
||||
public static readonly TimeSpan ReceiptFloor = TimeSpan.FromMinutes(10);
|
||||
public static readonly TimeSpan MaxCallerWait = TimeSpan.FromMinutes(5);
|
||||
|
||||
// Enable retries pull-and-recompute this many times before giving up on a store that will not hold still.
|
||||
public const int MaxEnableAttempts = 8;
|
||||
|
||||
public const int NotAuthorized = -1;
|
||||
|
||||
|
|
@ -59,13 +70,16 @@ namespace GameCult.Brokkr
|
|||
private readonly BrokkrCommandPolicy policy;
|
||||
private readonly Func<BrokkrUnityCommand, BrokkrUnityCommandReceipt> execute;
|
||||
private readonly IBrokkrCommandStore store;
|
||||
private readonly Func<DateTimeOffset> clock;
|
||||
|
||||
public BrokkrCommandDrain(
|
||||
BrokkrCommandLedger ledger,
|
||||
BrokkrCommandPolicy policy,
|
||||
Func<BrokkrUnityCommand, BrokkrUnityCommandReceipt> execute,
|
||||
IBrokkrCommandStore store)
|
||||
IBrokkrCommandStore store,
|
||||
Func<DateTimeOffset> clock = null)
|
||||
{
|
||||
this.clock = clock ?? (() => DateTimeOffset.UtcNow);
|
||||
this.ledger = ledger ?? throw new ArgumentNullException(nameof(ledger));
|
||||
this.policy = policy ?? throw new ArgumentNullException(nameof(policy));
|
||||
this.execute = execute ?? throw new ArgumentNullException(nameof(execute));
|
||||
|
|
@ -74,6 +88,10 @@ namespace GameCult.Brokkr
|
|||
|
||||
// The operator turned the sink on. Returns how many waiting intents it expired. The caller keeps the token
|
||||
// and passes it to every DrainAsync; nothing runs before this returns.
|
||||
//
|
||||
// The commit is conditional on the store being what this editor just observed, so an intent that lands
|
||||
// between the pull and the commit makes it refuse; the loop then pulls again and the newcomer is expired
|
||||
// with the rest. If writers never let it settle, it throws and the sink stays off.
|
||||
public async Task<int> EnableAsync(string token)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(token))
|
||||
|
|
@ -81,21 +99,32 @@ namespace GameCult.Brokkr
|
|||
throw new ArgumentException("A sink token is required.", nameof(token));
|
||||
}
|
||||
|
||||
var waiting = ledger.Pending(int.MaxValue);
|
||||
var expired = new BrokkrUnityCommandReceipt[waiting.Length];
|
||||
for (var index = 0; index < waiting.Length; index++)
|
||||
for (var attempt = 0; attempt < MaxEnableAttempts; attempt++)
|
||||
{
|
||||
expired[index] = Receipt(waiting[index], ExpiredStatus,
|
||||
"This intent was already in the store when the operator enabled the command sink, so the "
|
||||
+ "operator never saw it. It was not run.");
|
||||
// Our own staged writes first (a conditional commit refuses a dirty store), then everyone else's.
|
||||
await store.FlushAsync();
|
||||
await ledger.PullAsync();
|
||||
var waiting = ledger.Pending(int.MaxValue);
|
||||
var expired = new BrokkrUnityCommandReceipt[waiting.Length];
|
||||
for (var index = 0; index < waiting.Length; index++)
|
||||
{
|
||||
expired[index] = Receipt(waiting[index], ExpiredStatus,
|
||||
"This intent was already in the store when the operator enabled the command sink, so the "
|
||||
+ "operator never saw it. It was not run.");
|
||||
}
|
||||
|
||||
var marker = new BrokkrSinkEnabled { token = token, enabledAt = DateTime.UtcNow.ToString("O") };
|
||||
if (await store.TryCommitEnableAsync(expired, marker))
|
||||
{
|
||||
await store.FlushAsync();
|
||||
return waiting.Length;
|
||||
}
|
||||
}
|
||||
|
||||
await store.CommitEnableAsync(
|
||||
expired, new BrokkrSinkEnabled { token = token, enabledAt = DateTime.UtcNow.ToString("O") });
|
||||
await store.FlushAsync();
|
||||
return waiting.Length;
|
||||
throw new InvalidOperationException(
|
||||
$"The command store kept changing under the enable ({MaxEnableAttempts} attempts): something is writing "
|
||||
+ "intents continuously. Stop it and enable again; the sink stays off.");
|
||||
}
|
||||
|
||||
// Returns how many intents it answered, or NotAuthorized when this store holds no enable marker for the
|
||||
// token: the caller must treat the sink as off until the operator enables it again.
|
||||
public async Task<int> DrainAsync(string token)
|
||||
|
|
@ -144,7 +173,7 @@ namespace GameCult.Brokkr
|
|||
}
|
||||
|
||||
// One commit takes each intent with its receipt: an intent without its receipt would run again.
|
||||
var pruned = ledger.Prunable(RetainedReceipts, MaxPrunedPerTick);
|
||||
var pruned = ledger.Prunable(RetainedReceipts, MaxPrunedPerTick, clock() - ReceiptFloor);
|
||||
if (pruned.Length > 0)
|
||||
{
|
||||
await store.DeleteAnsweredAsync(pruned);
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ using GameCult.Caching;
|
|||
namespace GameCult.Brokkr
|
||||
{
|
||||
// Decides which Unity command intents still await execution. A command id is single-use: any receipt that
|
||||
// answers an id retires it for good, whatever is later written under that id. The ledger holds no state of its
|
||||
// answers an id retires it for as long as the receipt exists (BrokkrCommandDrain prunes old ones), whatever is later written under that id. The ledger holds no state of its
|
||||
// own; the answer is derived from the cache on every call, so there is no second copy that could drift.
|
||||
public sealed class BrokkrCommandLedger
|
||||
{
|
||||
|
|
@ -40,25 +40,28 @@ namespace GameCult.Brokkr
|
|||
}
|
||||
|
||||
// Up to max unanswered intents, oldest first, from one pass over the store. Stored-at only orders them; no
|
||||
// decision anywhere compares it with a clock.
|
||||
// decision anywhere compares it with a clock. A document counts only under the key its own id names, so an
|
||||
// intent cannot run twice by being stored under two keys, and a receipt cannot retire another key's intent.
|
||||
public BrokkrUnityCommand[] Pending(int max)
|
||||
{
|
||||
var stored = cache.AllStoredDocuments.ToArray();
|
||||
var answered = new HashSet<string>(
|
||||
stored.Select(entry => entry.Document).OfType<BrokkrUnityCommandReceipt>()
|
||||
.Select(receipt => receipt.commandId)
|
||||
.Where(id => !string.IsNullOrWhiteSpace(id)),
|
||||
stored.Where(entry => entry.Document is BrokkrUnityCommandReceipt receipt
|
||||
&& !string.IsNullOrWhiteSpace(receipt.commandId)
|
||||
&& entry.Key.Value == ReceiptKeyPrefix + receipt.commandId)
|
||||
.Select(entry => ((BrokkrUnityCommandReceipt)entry.Document).commandId),
|
||||
StringComparer.Ordinal);
|
||||
|
||||
return stored
|
||||
.Where(entry => entry.Document is BrokkrUnityCommand)
|
||||
.Where(entry => entry.Document is BrokkrUnityCommand candidate
|
||||
&& !string.IsNullOrWhiteSpace(candidate.commandId)
|
||||
&& entry.Key.Value == CommandKeyPrefix + candidate.commandId
|
||||
&& !answered.Contains(candidate.commandId))
|
||||
.OrderBy(entry => entry.StoredAt, StringComparer.Ordinal)
|
||||
.Select(entry => (BrokkrUnityCommand)entry.Document)
|
||||
.Where(candidate => !string.IsNullOrWhiteSpace(candidate.commandId) && !answered.Contains(candidate.commandId))
|
||||
.Take(max)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
// True when the store holds the enable marker for this token (see BrokkrCommandDrain).
|
||||
public bool SinkAuthorized(string token) =>
|
||||
!string.IsNullOrWhiteSpace(token)
|
||||
|
|
@ -75,19 +78,29 @@ namespace GameCult.Brokkr
|
|||
&& !string.IsNullOrWhiteSpace(receipt.commandId))
|
||||
.ToArray();
|
||||
|
||||
// The ids of the oldest answered intents beyond the newest keep receipts, at most max. Receipts are ordered by
|
||||
// their own stored-at, all minted by the editor, so one clock orders them. An attempt in flight is never listed.
|
||||
public string[] Prunable(int keep, int max)
|
||||
// The ids of the oldest answered intents beyond the newest keep receipts, at most max, and only those whose
|
||||
// receipt was stored at or before notAfter (see BrokkrCommandDrain.ReceiptFloor). Receipts are ordered by
|
||||
// their own stored-at, all minted by the editor, so one clock orders and ages them. An attempt in flight is
|
||||
// never listed.
|
||||
public string[] Prunable(int keep, int max, DateTimeOffset notAfter)
|
||||
{
|
||||
var receipts = cache.AllStoredDocuments
|
||||
.Where(entry => entry.Document is BrokkrUnityCommandReceipt receipt
|
||||
&& !string.IsNullOrWhiteSpace(receipt.commandId)
|
||||
&& entry.Key.Value == ReceiptKeyPrefix + receipt.commandId
|
||||
&& !string.Equals(receipt.status, BrokkrCommandDrain.AttemptedStatus, StringComparison.Ordinal))
|
||||
.OrderBy(entry => entry.StoredAt, StringComparer.Ordinal)
|
||||
.Select(entry => ((BrokkrUnityCommandReceipt)entry.Document).commandId)
|
||||
.ToArray();
|
||||
var excess = receipts.Length - keep;
|
||||
return excess <= 0 ? Array.Empty<string>() : receipts.Take(Math.Min(excess, max)).ToArray();
|
||||
return excess <= 0
|
||||
? Array.Empty<string>()
|
||||
: receipts.Take(excess)
|
||||
.TakeWhile(entry => !DateTimeOffset.TryParse(entry.StoredAt, System.Globalization.CultureInfo.InvariantCulture,
|
||||
System.Globalization.DateTimeStyles.RoundtripKind, out var stored)
|
||||
|| stored <= notAfter)
|
||||
.Take(max)
|
||||
.Select(entry => ((BrokkrUnityCommandReceipt)entry.Document).commandId)
|
||||
.ToArray();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
#nullable disable
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using GameCult.Caching;
|
||||
|
||||
namespace GameCult.Brokkr
|
||||
{
|
||||
// The drain's real store: every write the editor makes to the command lane is one CultCache commit, so what the
|
||||
// scenario harness runs is the code the editor ships. A commit costs one record write per record and no more;
|
||||
// durability is FlushAsync, which batches share.
|
||||
public sealed class BrokkrCommandStore : IBrokkrCommandStore
|
||||
{
|
||||
private readonly CultCache cache;
|
||||
|
||||
public BrokkrCommandStore(CultCache cache)
|
||||
{
|
||||
this.cache = cache ?? throw new ArgumentNullException(nameof(cache));
|
||||
}
|
||||
|
||||
public Task PutReceiptAsync(BrokkrUnityCommandReceipt receipt)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(receipt.commandId))
|
||||
{
|
||||
throw new ArgumentException("A receipt must name the command it answers.", nameof(receipt));
|
||||
}
|
||||
|
||||
return Commit(batch => batch.Upsert(
|
||||
receipt, new CultRecordHandle<BrokkrUnityCommandReceipt>(BrokkrCommandLedger.ReceiptKey(receipt.commandId))));
|
||||
}
|
||||
|
||||
public Task DeleteReceiptAsync(string commandId) =>
|
||||
Commit(batch => batch.Remove(BrokkrCommandLedger.ReceiptKey(commandId)));
|
||||
|
||||
// Conditional on the store being exactly what this cache has observed (CultCacheBatch.ExpectUnchanged): an
|
||||
// intent that landed since the caller's pull makes the commit refuse, and false comes back so the caller
|
||||
// pulls and recomputes. Flush first; a store with staged writes cannot take a conditional commit.
|
||||
public Task<bool> TryCommitEnableAsync(BrokkrUnityCommandReceipt[] expired, BrokkrSinkEnabled marker)
|
||||
{
|
||||
var committed = cache.Commit(batch =>
|
||||
{
|
||||
foreach (var receipt in expired)
|
||||
{
|
||||
batch.Upsert(receipt, new CultRecordHandle<BrokkrUnityCommandReceipt>(BrokkrCommandLedger.ReceiptKey(receipt.commandId)));
|
||||
}
|
||||
|
||||
batch.Upsert(marker, new CultRecordHandle<BrokkrSinkEnabled>(BrokkrCommandLedger.SinkKey));
|
||||
batch.ExpectUnchanged();
|
||||
});
|
||||
return Task.FromResult(committed);
|
||||
}
|
||||
|
||||
public Task DeleteAnsweredAsync(string[] commandIds) => Commit(batch =>
|
||||
{
|
||||
foreach (var commandId in commandIds)
|
||||
{
|
||||
batch.Remove(BrokkrCommandLedger.CommandKey(commandId));
|
||||
batch.Remove(BrokkrCommandLedger.ReceiptKey(commandId));
|
||||
}
|
||||
});
|
||||
|
||||
public Task FlushAsync() => cache.FlushAsync();
|
||||
|
||||
private Task Commit(Action<CultCacheBatch> stage) =>
|
||||
cache.Commit(stage) ? Task.CompletedTask : throw new InvalidOperationException("The store refused the commit.");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
fileFormatVersion: 2
|
||||
guid: dc4412b669e04680b5352ce098b4e630
|
||||
|
|
@ -33,7 +33,10 @@ namespace GameCult.Brokkr
|
|||
|
||||
// Written by the editor, into the store, when the operator enables the command sink. The token is also kept in
|
||||
// the editor's own EditorPrefs; a store whose marker does not carry that token was not enabled by this editor,
|
||||
// so nothing in it runs (BrokkrCommandDrain). A caller that can only write intents cannot forge it.
|
||||
// so nothing in it runs (BrokkrCommandDrain). That stops a stale "enabled" from authorizing a re-cloned or
|
||||
// foreign store. It does not authenticate writers: anyone who can write .brokkr can write intents, and the
|
||||
// Allowed Actions policy is what limits them. Restoring a backup, a git checkout, or a sync-conflict copy of
|
||||
// this editor's own store carries a valid marker and can bring back intents that were already answered.
|
||||
[CultDocument("brokkr.unity.sink_enabled", "brokkr.unity.sink_enabled.v0")]
|
||||
[MessagePackObject]
|
||||
[Serializable]
|
||||
|
|
|
|||
|
|
@ -108,17 +108,25 @@ dotnet run --project F:/Projects/Brokkr/brokkr-command/Brokkr.Command.csproj --
|
|||
|
||||
Turning `Agent Commands` on expires every intent already in `.brokkr/`: each
|
||||
gets a receipt with `status: expired` and none runs, so enabling the sink
|
||||
cannot execute anything you did not see. Intents that arrive after the tick are
|
||||
fresh, whatever clock wrote them. With about 2000 waiting intents the tick
|
||||
takes several seconds (the store writes one receipt each); after that the editor
|
||||
handles at most 16 intents per update tick.
|
||||
cannot execute anything you did not see. The expiry is one commit that only
|
||||
lands if the store is exactly what the editor just read; an intent that arrives
|
||||
mid-enable makes it read again, and if writers never let the store settle
|
||||
(eight tries) the enable fails with a message and the sink stays off. Intents
|
||||
that arrive after the tick are fresh, whatever clock wrote them. With about
|
||||
2000 waiting intents the tick takes several seconds (the store writes one
|
||||
receipt each); after that the editor handles at most 16 intents per update
|
||||
tick.
|
||||
|
||||
The tick also writes an enable marker into `.brokkr/unity-editor.ccmp`, and the
|
||||
editor keeps the matching token in its own preferences. The sink runs only
|
||||
while the store carries that marker. If you re-clone the project, or copy a
|
||||
`.brokkr/` folder in, the marker is missing or foreign, so the editor turns the
|
||||
sink off and logs why; tick `Agent Commands` again to run intents from that
|
||||
store.
|
||||
store. The token is not authentication and does not identify who wrote an
|
||||
intent: anyone who can write `.brokkr/` can write intents, and `Allowed
|
||||
Actions` is what limits them. Restoring a backup, checking out `.brokkr/` from
|
||||
git, or a sync-conflict copy of the editor's own store carries a valid marker
|
||||
and can bring back intents that were already answered, which then run again.
|
||||
|
||||
Before the editor runs an intent it writes a receipt with `status: attempted`.
|
||||
If the editor stops, or cannot record the result, that intent is reported as
|
||||
|
|
@ -127,12 +135,16 @@ If the editor stops, or cannot record the result, that intent is reported as
|
|||
not idempotent. Re-sending `createGameObject` after an `interrupted` receipt
|
||||
can create the object twice; a domain reload during `refreshAssets` can also
|
||||
end as `interrupted` although the refresh completed. If the editor could not
|
||||
even write the `attempted` receipt, the intent never started and is retried on
|
||||
the next tick.
|
||||
|
||||
Receipts are kept for the newest 1000 answered intents; older intents are
|
||||
deleted together with their receipts, at most 128 per tick.
|
||||
write the `attempted` receipt, the intent normally never started and is retried
|
||||
on the next tick; if the store fails in a way that leaves the marker behind,
|
||||
it is reported as `interrupted` instead.
|
||||
|
||||
Receipts are kept for the newest 1000 answered intents, and no receipt is
|
||||
deleted before it is ten minutes old, whatever the count (`brokkr-command`
|
||||
refuses a `--wait-ms` over five minutes, so the receipt a caller waits on is
|
||||
always still there). Older intents beyond the count are deleted together with
|
||||
their receipts, at most 128 per tick. A `--command-id` is refused only while
|
||||
its receipt exists, so an id can be reused after its receipt has been pruned.
|
||||
`brokkr-command` exits 0 when the receipt is `accepted`, 1 for any other
|
||||
receipt status (`failed`, `denied`, `expired`, `interrupted`), and 2 for bad
|
||||
usage, a timeout, a store that cannot be opened, or any other failure. A
|
||||
|
|
|
|||
Loading…
Reference in a new issue