From 1480883a3b5a3b54a3d6cbf3088c56d48649b703 Mon Sep 17 00:00:00 2001 From: Heimdall Date: Sat, 14 Nov 2026 13:25:30 +0000 Subject: [PATCH 1/5] Test: operator settle of a stuck merge, the admin page and the sweeper's proof prune (red; SettleStuckAsync, PruneExpired and the page are skeletons) Co-Authored-By: Claude Sonnet 5.5 --- .../Features/Membership/AccountMergeSaga.cs | 36 +++ .../Features/Membership/MergeProofs.cs | 2 + src/Bifrost.Web/Pages/App/Merges/Index.cshtml | 2 + .../Pages/App/Merges/Index.cshtml.cs | 9 + .../AccountMergePostgresTests.cs | 2 +- .../AccountMergeSettlePageTests.cs | 160 ++++++++++ .../AccountMergeSettleTests.cs | 291 ++++++++++++++++++ tests/Bifrost.Web.Tests/MergeProofsTests.cs | 14 + 8 files changed, 515 insertions(+), 1 deletion(-) create mode 100644 src/Bifrost.Web/Pages/App/Merges/Index.cshtml create mode 100644 src/Bifrost.Web/Pages/App/Merges/Index.cshtml.cs create mode 100644 tests/Bifrost.Web.Tests/AccountMergeSettlePageTests.cs create mode 100644 tests/Bifrost.Web.Tests/AccountMergeSettleTests.cs diff --git a/src/Bifrost.Web/Features/Membership/AccountMergeSaga.cs b/src/Bifrost.Web/Features/Membership/AccountMergeSaga.cs index dbf91cc..a7662ba 100644 --- a/src/Bifrost.Web/Features/Membership/AccountMergeSaga.cs +++ b/src/Bifrost.Web/Features/Membership/AccountMergeSaga.cs @@ -5,6 +5,7 @@ using System.Text.RegularExpressions; using Bifrost.Web.Data; using Bifrost.Web.Domain; using Bifrost.Web.Features.Heimdall; +using Bifrost.Web.Features.Shared; using Microsoft.AspNetCore.DataProtection; using Microsoft.EntityFrameworkCore; using Npgsql; @@ -70,6 +71,31 @@ public abstract record UndoOutcome public sealed record ProofAccountMismatch : UndoOutcome; } +/// The two ways an operator may end a stuck leg: the forward move is failed, or the undo is given up and the merge stands. +public enum SettleChoice +{ + FailForward, + AbandonUndo +} + +/// The closed answer to an operator's settle. Only Settled wrote. +public abstract record SettleOutcome +{ + private SettleOutcome() + { + } + + public sealed record Settled : SettleOutcome; + + /// The merge is missing, in another state than the choice ends, or not stuck on one of the settleable outcomes. + public sealed record NotStuck : SettleOutcome; + + public sealed record WrongMoveId : SettleOutcome; + + /// The State token went to another writer first; the other transition stands and nothing was changed. + public sealed record Lost : SettleOutcome; +} + /// /// The one owner of an account merge's lifecycle outside the ledger's Committed, Final and Undone, and the only caller of /// . It inserts Confirmed (the consent record, with the Heimdall bearers sealed and the move @@ -86,12 +112,16 @@ public sealed partial class AccountMergeSaga( AccountMergeLedger ledger, HeimdallIdentityMoveClient mover, IDataProtectionProvider dataProtection, + AuditTrailService audit, TimeProvider timeProvider, ILogger logger) { public const string MergeInFlight = "merge_in_flight"; public const string ReceiptMismatch = "receipt_mismatch"; public const string ProofsUnreadable = "proofs_unreadable"; + public const string MoveIdConflict = "move_id_conflict"; + public const string OperatorFailed = "operator_failed"; + public const string UndoOperatorAbandoned = "undo_operator_abandoned"; private const string LiveMergeIndex = "UX_AccountMerges_FoldedUserAccountId_Live"; private const int MaxProofs = 8; @@ -399,6 +429,12 @@ public sealed partial class AccountMergeSaga( return advanced; } + // ---- operator settle ---- + + public Task SettleStuckAsync( + Guid mergeId, SettleChoice choice, string typedMoveId, Guid operatorUserAccountId, CancellationToken cancellationToken) => + throw new NotImplementedException(); + // ---- helpers ---- /// The merge as stored now: a row this context already tracks is reloaded, never trusted. diff --git a/src/Bifrost.Web/Features/Membership/MergeProofs.cs b/src/Bifrost.Web/Features/Membership/MergeProofs.cs index 2faefbf..a9d614f 100644 --- a/src/Bifrost.Web/Features/Membership/MergeProofs.cs +++ b/src/Bifrost.Web/Features/Membership/MergeProofs.cs @@ -186,6 +186,8 @@ public sealed class MergeProofs(TimeProvider timeProvider) } } + public void PruneExpired() => throw new NotImplementedException(); + private static List UnprovedProviders(Side side, IEnumerable providers) => [.. providers.Distinct(StringComparer.Ordinal).Where(x => !side.Proofs.ContainsKey(x))]; diff --git a/src/Bifrost.Web/Pages/App/Merges/Index.cshtml b/src/Bifrost.Web/Pages/App/Merges/Index.cshtml new file mode 100644 index 0000000..861f53a --- /dev/null +++ b/src/Bifrost.Web/Pages/App/Merges/Index.cshtml @@ -0,0 +1,2 @@ +@page +@model Bifrost.Web.Pages.App.Merges.IndexModel diff --git a/src/Bifrost.Web/Pages/App/Merges/Index.cshtml.cs b/src/Bifrost.Web/Pages/App/Merges/Index.cshtml.cs new file mode 100644 index 0000000..0506cb7 --- /dev/null +++ b/src/Bifrost.Web/Pages/App/Merges/Index.cshtml.cs @@ -0,0 +1,9 @@ +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.RazorPages; + +namespace Bifrost.Web.Pages.App.Merges; + +public sealed class IndexModel : PageModel +{ + public IActionResult OnGet() => NotFound(); +} diff --git a/tests/Bifrost.Web.Tests/AccountMergePostgresTests.cs b/tests/Bifrost.Web.Tests/AccountMergePostgresTests.cs index 57d8cab..06981a6 100644 --- a/tests/Bifrost.Web.Tests/AccountMergePostgresTests.cs +++ b/tests/Bifrost.Web.Tests/AccountMergePostgresTests.cs @@ -728,7 +728,7 @@ public sealed class AccountMergePostgresTests(MergePostgres postgres) : IClassFi public FakeHeimdall Fake { get; } public AccountMergeSaga Saga(AccountMergeLedgerTests.World world, BifrostDbContext db) => - new(db, world.Ledger(db), _mover, _keys, world.Clock, NullLogger.Instance); + new(db, world.Ledger(db), _mover, _keys, new AuditTrailService(db, world.Clock), world.Clock, NullLogger.Instance); /// A world account with one login at the fake Heimdall and a weak proof through it. public MergeProof Proof(UserAccount account, string provider) => diff --git a/tests/Bifrost.Web.Tests/AccountMergeSettlePageTests.cs b/tests/Bifrost.Web.Tests/AccountMergeSettlePageTests.cs new file mode 100644 index 0000000..81e53ab --- /dev/null +++ b/tests/Bifrost.Web.Tests/AccountMergeSettlePageTests.cs @@ -0,0 +1,160 @@ +using System.Net; +using Bifrost.Web.Data; +using Bifrost.Web.Domain; +using Bifrost.Web.Features.Membership; +using Bifrost.Web.Tests.Support; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using static Bifrost.Web.Tests.AccountMergeSagaTests; +using static Bifrost.Web.Tests.AccountMergeSettleTests; + +namespace Bifrost.Web.Tests; + +/// The operator's page for stuck merges, from browser sessions of the real host: admin only, the stored move id shown as the confirmation, results by closed code. +public sealed class AccountMergeSettlePageTests +{ + private const string Url = "/App/Merges"; + + private sealed class World : IDisposable + { + public Rig Rig { get; } = new(realSessions: true); + + public Party Admin { get; private set; } = null!; + + public string AdminSession { get; private set; } = string.Empty; + + public string MemberSession { get; private set; } = string.Empty; + + public Stuck Forward { get; private set; } = null!; + + public static async Task Make() + { + var world = new World(); + var rig = world.Rig; + world.Admin = await rig.AccountAsync("operator", "github"); + var member = await rig.AccountAsync("member", "github"); + await rig.InScopeAsync(async sp => + { + var db = sp.GetRequiredService(); + var membership = await db.Memberships.SingleAsync(x => x.UserAccountId == world.Admin.Id); + membership.IsPlatformAdmin = true; + await db.SaveChangesAsync(); + return 0; + }); + world.AdminSession = await rig.Host.SignInSessionAsync(world.Admin.HeimdallId); + world.MemberSession = await rig.Host.SignInSessionAsync(member.HeimdallId); + world.Forward = await ConfirmedStuckAsync(rig, AccountMergeSaga.ReceiptMismatch); + return world; + } + + public void Dispose() => Rig.Dispose(); + + public Task<(HttpResponseMessage Response, string Html)> Get(string session, string query = "") => MergePage.GetAsync(Rig.Host, session, query, Url); + + public async Task Post(string session, string handler, string mergeId, string moveId, MergePage.Antiforgery? token = null) => + await MergePage.PostAsync(Rig.Host, session, token ?? await MergePage.TokenAsync(Rig.Host, session, MergePage.Url), handler, + new Dictionary { ["mergeId"] = mergeId, ["moveId"] = moveId }, Url); + } + + private static bool Refused(HttpResponseMessage response) => + response.StatusCode == HttpStatusCode.Forbidden || (response.StatusCode == HttpStatusCode.Redirect && (response.Headers.Location?.OriginalString.Contains("AccessDenied", StringComparison.OrdinalIgnoreCase) ?? false)); + + [Fact] + public async Task A_member_who_is_not_a_platform_admin_is_refused_on_the_listing_and_on_both_posts_and_nothing_changes() + { + using var w = await World.Make(); + var merge = await w.Rig.MergeAsync(w.Forward.Id); + var requests = w.Rig.Moves.Requests.Count; + + var (get, html) = await w.Get(w.MemberSession); + Assert.True(Refused(get), $"listing gave {(int)get.StatusCode}"); + Assert.DoesNotContain(merge.HeimdallMoveId!, html); + Assert.True(Refused(await w.Post(w.MemberSession, "FailForward", w.Forward.Id.ToString(), merge.HeimdallMoveId!))); + Assert.True(Refused(await w.Post(w.MemberSession, "AbandonUndo", w.Forward.Id.ToString(), merge.HeimdallMoveId!))); + + var after = await w.Rig.MergeAsync(w.Forward.Id); + Assert.Equal(AccountMergeState.Confirmed, after.State); + Assert.Equal(AccountMergeSaga.ReceiptMismatch, after.LastOutcome); + Assert.Equal(requests, w.Rig.Moves.Requests.Count); + } + + [Fact] + public async Task The_listing_shows_stuck_merges_with_their_stored_move_id_and_no_Heimdall_account_or_provider_user_id() + { + using var w = await World.Make(); + var stuck = w.Forward; + var merge = await w.Rig.MergeAsync(stuck.Id); + // Heimdall unreachable for a long time: listed, though only the three stuck outcomes can be ended. + var slow = await ConfirmedStuckAsync(w.Rig, AccountMergeSaga.MoveIdConflict); + var slowMerge = await w.Rig.MergeAsync(slow.Id); + var waiting = await ConfirmedStuckAsync(w.Rig, "transport"); + await EditMerge(w.Rig, waiting.Id, x => x.Attempts = 8); + var healthy = await ConfirmedStuckAsync(w.Rig, "confirmed"); // one waiting for its first replay: not stuck + await EditMerge(w.Rig, healthy.Id, x => x.Attempts = 1); + var undo = await UndoStuckAsync(w.Rig); + var undoMerge = await w.Rig.MergeAsync(undo.Id); + + var (response, html) = await w.Get(w.AdminSession); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Contains(stuck.Id.ToString(), html); + Assert.Contains(merge.HeimdallMoveId!, html); + Assert.Contains(slow.Id.ToString(), html); + Assert.Contains(slowMerge.HeimdallMoveId!, html); + Assert.Contains(waiting.Id.ToString(), html); + Assert.Contains(undo.Id.ToString(), html); + Assert.Contains(undoMerge.HeimdallReverseMoveId!, html); + Assert.DoesNotContain(undoMerge.HeimdallMoveId!, html); // the leg in flight is the reversal + Assert.DoesNotContain(healthy.Id.ToString(), html); + foreach (var party in new[] { stuck.Survivor, stuck.Folded, slow.Survivor, slow.Folded, undo.Survivor, undo.Folded }) + { + Assert.DoesNotContain(party.HeimdallId, html); + Assert.DoesNotContain(party.Id.ToString(), html); + Assert.DoesNotContain(party.Logins.Values.First(), html); + } + } + + [Fact] + public async Task A_platform_admin_ends_a_stuck_leg_by_typing_its_move_id_and_a_wrong_one_changes_nothing() + { + using var w = await World.Make(); + var merge = await w.Rig.MergeAsync(w.Forward.Id); + var token = await MergePage.TokenAsync(w.Rig.Host, w.AdminSession, Url); + var requests = w.Rig.Moves.Requests.Count; + + var wrong = await w.Post(w.AdminSession, "FailForward", w.Forward.Id.ToString(), "0123456789abcdef0123456789abcdef", token); + Assert.Equal(Url + "?result=wrong_move_id", MergePage.Location(wrong)); + Assert.Equal(AccountMergeState.Confirmed, (await w.Rig.MergeAsync(w.Forward.Id)).State); + + var otherLeg = await w.Post(w.AdminSession, "AbandonUndo", w.Forward.Id.ToString(), merge.HeimdallMoveId!, token); + Assert.Equal(Url + "?result=not_stuck", MergePage.Location(otherLeg)); + + var done = await w.Post(w.AdminSession, "FailForward", w.Forward.Id.ToString(), merge.HeimdallMoveId!, token); + Assert.Equal(Url + "?result=settled", MergePage.Location(done)); + var after = await w.Rig.MergeAsync(w.Forward.Id); + Assert.Equal(AccountMergeState.Failed, after.State); + await w.Rig.InScopeAsync(async sp => + { + var audit = await sp.GetRequiredService().AuditEvents.AsNoTracking().SingleAsync(x => x.EntityId == w.Forward.Id.ToString()); + Assert.Equal(w.Admin.Id, audit.ActorUserAccountId); + return 0; + }); + Assert.Equal(requests, w.Rig.Moves.Requests.Count); // the page calls no Heimdall + + var (_, html) = await w.Get(w.AdminSession, "?result=settled"); + Assert.DoesNotContain(w.Forward.Id.ToString(), html); // no longer stuck, so no longer listed + } + + [Fact] + public async Task A_platform_admin_gives_up_a_stuck_undo_by_typing_the_reverse_move_id() + { + using var w = await World.Make(); + var undo = await UndoStuckAsync(w.Rig); + var merge = await w.Rig.MergeAsync(undo.Id); + + var done = await w.Post(w.AdminSession, "AbandonUndo", undo.Id.ToString(), merge.HeimdallReverseMoveId!, await MergePage.TokenAsync(w.Rig.Host, w.AdminSession, Url)); + + Assert.Equal(Url + "?result=settled", MergePage.Location(done)); + Assert.Equal(AccountMergeState.Committed, (await w.Rig.MergeAsync(undo.Id)).State); + } +} diff --git a/tests/Bifrost.Web.Tests/AccountMergeSettleTests.cs b/tests/Bifrost.Web.Tests/AccountMergeSettleTests.cs new file mode 100644 index 0000000..608d6a4 --- /dev/null +++ b/tests/Bifrost.Web.Tests/AccountMergeSettleTests.cs @@ -0,0 +1,291 @@ +using Bifrost.Web.Data; +using Bifrost.Web.Domain; +using Bifrost.Web.Features.Heimdall; +using Bifrost.Web.Features.Membership; +using Bifrost.Web.Tests.Support; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using static Bifrost.Web.Tests.AccountMergeSagaTests; + +namespace Bifrost.Web.Tests; + +/// +/// The operator's settle of a stuck merge (ruling merge-stuck-end-only-typed-id): two ends, FailForward and AbandonUndo, only from a +/// stuck leg and only with the leg's stored move id typed back, one audit row, never an adopt. The saga rig of AccountMergeSagaTests. +/// +public sealed class AccountMergeSettleTests +{ + private static readonly Guid Operator = Guid.NewGuid(); + + internal sealed record Stuck(Guid Id, Party Survivor, Party Folded); + + /// A Confirmed merge that is stuck on : the two the saga records itself are reached through the real step, the third is the stored Unknown reason. + internal static async Task ConfirmedStuckAsync(Rig rig, string outcome) + { + var survivor = await rig.AccountAsync("survivor", "twitch"); + var folded = await rig.AccountAsync("folded", "discord"); + var id = await rig.ConfirmedAsync(survivor, folded, rig.Weak(survivor, "twitch"), rig.Weak(folded, "discord")); + switch (outcome) + { + case AccountMergeSaga.ReceiptMismatch: + await Edit(rig, x => x.UserAccounts.Single(a => a.Id == survivor.Id).HeimdallAccountId = HeimdallShapes.NewAccountId()); + await rig.AdvanceAsync(id); + break; + case AccountMergeSaga.ProofsUnreadable: + await EditMerge(rig, id, x => x.SealedProofs = "CfDJ8-not-a-protected-payload"); + await rig.AdvanceAsync(id); + break; + default: + await EditMerge(rig, id, x => { x.LastOutcome = outcome; x.Attempts = 3; x.NextAttemptUtc = rig.Clock.Now.AddMinutes(5); }); + break; + } + + Assert.Equal(outcome, (await rig.MergeAsync(id)).LastOutcome); + return new Stuck(id, survivor, folded); + } + + /// A merge in UndoRequested whose reversal got a receipt for another account. + internal static async Task UndoStuckAsync(Rig rig) + { + var survivor = await rig.AccountAsync("survivor", "twitch"); + var folded = await rig.AccountAsync("folded", "discord"); + var id = await rig.ConfirmedAsync(survivor, folded, rig.Weak(survivor, "twitch"), rig.Weak(folded, "discord")); + await rig.AdvanceAsync(id); + var undo = await rig.InScopeAsync(sp => sp.GetRequiredService().RequestUndoAsync(id, survivor.Id, [rig.Moved(survivor, folded, "discord")], default)); + Assert.IsType(undo); + await Edit(rig, x => x.UserAccounts.Single(a => a.Id == folded.Id).HeimdallAccountId = HeimdallShapes.NewAccountId()); + await rig.AdvanceAsync(id); + var stuck = await rig.MergeAsync(id); + Assert.Equal(AccountMergeState.UndoRequested, stuck.State); + Assert.Equal(AccountMergeSaga.ReceiptMismatch, stuck.LastOutcome); + return new Stuck(id, survivor, folded); + } + + private static Task Edit(Rig rig, Action change) => rig.InScopeAsync(async sp => + { + var db = sp.GetRequiredService(); + change(db); + await db.SaveChangesAsync(); + return 0; + }); + + internal static Task EditMerge(Rig rig, Guid id, Action change) => Edit(rig, db => change(db.AccountMerges.Single(x => x.Id == id))); + + private static Task SettleAsync(Rig rig, Guid id, SettleChoice choice, string? typed) => + rig.InScopeAsync(sp => sp.GetRequiredService().SettleStuckAsync(id, choice, typed!, Operator, default)); + + private static Task> AuditAsync(Rig rig, Guid id) => + rig.InScopeAsync(sp => sp.GetRequiredService().AuditEvents.AsNoTracking().Where(x => x.EntityId == id.ToString()).ToListAsync()); + + // ---- FailForward ---- + + [Theory] + [InlineData("receipt_mismatch")] + [InlineData("proofs_unreadable")] + [InlineData("move_id_conflict")] + public async Task FailForward_ends_a_stuck_Confirmed_merge_as_Failed_with_the_proofs_wiped_the_folded_account_free_and_one_audit_row(string outcome) + { + using var rig = new Rig(); + var stuck = await ConfirmedStuckAsync(rig, outcome); + Assert.IsType(await rig.StandingAsync(stuck.Folded)); + var moveId = (await rig.MergeAsync(stuck.Id)).HeimdallMoveId!; + + Assert.IsType(await SettleAsync(rig, stuck.Id, SettleChoice.FailForward, moveId)); + + var merge = await rig.MergeAsync(stuck.Id); + Assert.Equal(AccountMergeState.Failed, merge.State); + Assert.Equal(AccountMergeSaga.OperatorFailed, merge.FailureCode); + Assert.Equal(rig.Clock.Now, merge.FailedUtc); + Assert.Null(merge.SealedProofs); + Assert.Null(merge.NextAttemptUtc); + Assert.Null(await rig.StandingAsync(stuck.Folded)); + Assert.Null(await rig.StandingAsync(stuck.Survivor)); + Assert.Empty(await rig.JournalAsync(stuck.Id)); + Assert.Equal(0, await rig.AdvanceDueAsync()); // the sweeper does not pick a settled merge up again + var audit = Assert.Single(await AuditAsync(rig, stuck.Id)); + Assert.Equal("merge.settled", audit.Action); + Assert.Equal(Operator, audit.ActorUserAccountId); + Assert.Contains("FailForward", audit.Detail); + // The row names the choice and the merge: nothing of either side. + Assert.DoesNotContain(stuck.Survivor.HeimdallId, audit.Detail); + Assert.DoesNotContain(stuck.Folded.HeimdallId, audit.Detail); + Assert.DoesNotContain(moveId, audit.Detail); + Assert.DoesNotContain(stuck.Survivor.Id.ToString(), audit.Detail); + Assert.DoesNotContain(stuck.Folded.Id.ToString(), audit.Detail); + } + + [Fact] + public async Task A_settled_merge_can_be_confirmed_again_for_the_same_pair() + { + using var rig = new Rig(); + var stuck = await ConfirmedStuckAsync(rig, AccountMergeSaga.ReceiptMismatch); + await EditHeimdallBack(rig, stuck); + Assert.IsType(await SettleAsync(rig, stuck.Id, SettleChoice.FailForward, (await rig.MergeAsync(stuck.Id)).HeimdallMoveId)); + + Assert.IsType(await rig.ConfirmAsync(stuck.Survivor, stuck.Folded, rig.Weak(stuck.Survivor, "twitch"), rig.Weak(stuck.Folded, "discord"))); + } + + private static Task EditHeimdallBack(Rig rig, Stuck stuck) => + Edit(rig, db => db.UserAccounts.Single(x => x.Id == stuck.Survivor.Id).HeimdallAccountId = stuck.Survivor.HeimdallId); + + // ---- AbandonUndo ---- + + [Fact] + public async Task AbandonUndo_returns_a_stuck_UndoRequested_merge_to_Committed_with_the_grace_the_proofs_wiped_and_one_audit_row() + { + using var rig = new Rig(); + var stuck = await UndoStuckAsync(rig); + var before = await rig.MergeAsync(stuck.Id); + + Assert.IsType(await SettleAsync(rig, stuck.Id, SettleChoice.AbandonUndo, before.HeimdallReverseMoveId)); + + var merge = await rig.MergeAsync(stuck.Id); + Assert.Equal(AccountMergeState.Committed, merge.State); + Assert.Equal(AccountMergeSaga.UndoOperatorAbandoned, merge.LastOutcome); + Assert.Null(merge.SealedProofs); + Assert.Null(merge.NextAttemptUtc); + Assert.Equal(before.CommittedUtc, merge.CommittedUtc); + Assert.Equal(before.GraceEndsUtc, merge.GraceEndsUtc); + Assert.IsType(await rig.StandingAsync(stuck.Folded)); + Assert.Equal(0, await rig.AdvanceDueAsync()); + var audit = Assert.Single(await AuditAsync(rig, stuck.Id)); + Assert.Equal("merge.settled", audit.Action); + Assert.Contains("AbandonUndo", audit.Detail); + Assert.DoesNotContain(before.HeimdallReverseMoveId!, audit.Detail); + + rig.Clock.Advance(AccountMergeLedger.Grace + TimeSpan.FromMinutes(1)); + await rig.InScopeAsync(sp => sp.GetRequiredService().FinalizeDueAsync(rig.Clock.Now, default)); + Assert.Equal(AccountMergeState.Final, (await rig.MergeAsync(stuck.Id)).State); + } + + // ---- refusals write nothing ---- + + [Fact] + public async Task A_merge_in_any_other_state_or_with_another_LastOutcome_is_NotStuck_and_nothing_is_written() + { + using var rig = new Rig(); + var survivor = await rig.AccountAsync("survivor", "twitch"); + var folded = await rig.AccountAsync("folded", "discord"); + var id = await rig.ConfirmedAsync(survivor, folded, rig.Weak(survivor, "twitch"), rig.Weak(folded, "discord")); + var moveId = (await rig.MergeAsync(id)).HeimdallMoveId!; + + // Confirmed and healthy ("confirmed"), then waiting on Heimdall ("transport"): not stuck. + Assert.IsType(await SettleAsync(rig, id, SettleChoice.FailForward, moveId)); + await EditMerge(rig, id, x => { x.LastOutcome = "transport"; x.Attempts = 9; }); + Assert.IsType(await SettleAsync(rig, id, SettleChoice.FailForward, moveId)); + var snapshot = await rig.MergeAsync(id); + Assert.Equal(AccountMergeState.Confirmed, snapshot.State); + Assert.NotNull(snapshot.SealedProofs); + + // A stuck outcome does not make the other leg's choice valid. + await EditMerge(rig, id, x => x.LastOutcome = AccountMergeSaga.ReceiptMismatch); + Assert.IsType(await SettleAsync(rig, id, SettleChoice.AbandonUndo, moveId)); + + // Heimdall answered Moved: HeimdallMoved and Committed are never settleable, whatever LastOutcome says. + foreach (var state in new[] { AccountMergeState.HeimdallMoved, AccountMergeState.Committed, AccountMergeState.HeimdallReversed, AccountMergeState.Undone, AccountMergeState.Final, AccountMergeState.Failed }) + { + await EditMerge(rig, id, x => { x.State = state; x.LastOutcome = AccountMergeSaga.ReceiptMismatch; }); + Assert.IsType(await SettleAsync(rig, id, SettleChoice.FailForward, moveId)); + Assert.IsType(await SettleAsync(rig, id, SettleChoice.AbandonUndo, moveId)); + Assert.Equal(state, (await rig.MergeAsync(id)).State); + } + + Assert.IsType(await SettleAsync(rig, Guid.NewGuid(), SettleChoice.FailForward, moveId)); + Assert.Empty(await AuditAsync(rig, id)); + } + + [Fact] + public async Task A_move_id_that_is_not_the_stored_one_of_the_leg_is_WrongMoveId_and_nothing_is_written() + { + using var rig = new Rig(); + var stuck = await ConfirmedStuckAsync(rig, AccountMergeSaga.ReceiptMismatch); + var merge = await rig.MergeAsync(stuck.Id); + + foreach (var typed in new[] { "", " ", merge.HeimdallMoveId!.ToUpperInvariant(), merge.HeimdallMoveId + " ", merge.HeimdallMoveId![..31], stuck.Id.ToString("N"), Guid.NewGuid().ToString("N") }) + { + Assert.IsType(await SettleAsync(rig, stuck.Id, SettleChoice.FailForward, typed)); + } + + Assert.IsType(await SettleAsync(rig, stuck.Id, SettleChoice.FailForward, null)); + var after = await rig.MergeAsync(stuck.Id); + Assert.Equal(AccountMergeState.Confirmed, after.State); + Assert.Equal(AccountMergeSaga.ReceiptMismatch, after.LastOutcome); + Assert.Empty(await AuditAsync(rig, stuck.Id)); + } + + [Fact] + public async Task The_undo_leg_is_confirmed_by_the_reverse_move_id_not_the_forward_one() + { + using var rig = new Rig(); + var stuck = await UndoStuckAsync(rig); + var merge = await rig.MergeAsync(stuck.Id); + + Assert.IsType(await SettleAsync(rig, stuck.Id, SettleChoice.AbandonUndo, merge.HeimdallMoveId)); + + Assert.Equal(AccountMergeState.UndoRequested, (await rig.MergeAsync(stuck.Id)).State); + Assert.Empty(await AuditAsync(rig, stuck.Id)); + } + + // ---- the race ---- + + [Fact] + public async Task A_settle_that_races_AdvanceAsync_and_loses_on_the_State_token_changes_nothing() + { + using var rig = new Rig(); + // Heimdall answered move_id_conflict, so the sweeper keeps replaying: the day its proofs verify, the replay lands. + var stuck = await ConfirmedStuckAsync(rig, AccountMergeSaga.MoveIdConflict); + await EditMerge(rig, stuck.Id, x => x.NextAttemptUtc = null); + var moveId = (await rig.MergeAsync(stuck.Id)).HeimdallMoveId!; + var raced = false; + rig.Host.Fault.When = db => + { + if (raced || !db.ChangeTracker.Entries().Any(x => x.Entity.State == AccountMergeState.Failed)) + { + return false; + } + + raced = true; // the settle has staged Failed and is about to save: the step function moves and commits first + Task.Run(() => rig.AdvanceAsync(stuck.Id)).GetAwaiter().GetResult(); + return false; + }; + + var outcome = await SettleAsync(rig, stuck.Id, SettleChoice.FailForward, moveId); + rig.Host.Fault.When = null; + + Assert.True(raced); + Assert.IsType(outcome); + Assert.Equal(AccountMergeState.Committed, (await rig.MergeAsync(stuck.Id)).State); + Assert.Empty((await rig.MergeAsync(stuck.Id)).FailureCode); + Assert.Empty(await AuditAsync(rig, stuck.Id)); + Assert.IsType(await rig.StandingAsync(stuck.Folded)); + } + + // ---- proof bearers are swept by the existing tick ---- + + [Fact] + public async Task The_sweeper_tick_drops_a_bearer_past_its_lifetime_with_no_page_access() + { + using var rig = new Rig(); + var proofs = rig.Host.Services.GetRequiredService(); + var actor = Guid.NewGuid(); + var proof = new MergeProof("BEARER-CANARY", HeimdallShapes.NewAccountId(), "discord"); + Assert.Equal(PutOutcome.Stored, proofs.Put(actor, MergeSide.Self, proof, Guid.NewGuid(), [])); + rig.Clock.Advance(MergeProofs.ProofLifetime + AccountMergeSweeper.Tick); + Assert.Equal(1, HeldActors(proofs)); // nobody has touched it since + + var sweeper = new AccountMergeSweeper(rig.Host.Services.GetRequiredService(), rig.Clock, NullLogger.Instance); + await sweeper.StartAsync(default); + var deadline = DateTime.UtcNow.AddSeconds(10); + while (DateTime.UtcNow < deadline && HeldActors(proofs) > 0) + { + await Task.Delay(50); + } + + await sweeper.StopAsync(default); + Assert.Equal(0, HeldActors(proofs)); + } + + private static int HeldActors(MergeProofs proofs) => + ((System.Collections.IDictionary)typeof(MergeProofs).GetField("_held", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)!.GetValue(proofs)!).Count; +} diff --git a/tests/Bifrost.Web.Tests/MergeProofsTests.cs b/tests/Bifrost.Web.Tests/MergeProofsTests.cs index 8982384..5c28e48 100644 --- a/tests/Bifrost.Web.Tests/MergeProofsTests.cs +++ b/tests/Bifrost.Web.Tests/MergeProofsTests.cs @@ -214,6 +214,20 @@ public sealed class MergeProofsTests Assert.Equal(0, HeldProofs()); } + [Fact] + public void PruneExpired_drops_an_expired_bearer_with_no_other_access_and_keeps_a_live_one() + { + PutSelf("discord", Login("discord")); + _clock.Advance(MergeProofs.ProofLifetime); + _proofs.PruneExpired(); + Assert.Equal(0, HeldProofs()); + + PutSelf("discord", Login("discord")); + _clock.Advance(MergeProofs.ProofLifetime - TimeSpan.FromSeconds(1)); + _proofs.PruneExpired(); + Assert.Equal(1, HeldProofs()); + } + [Fact] public void Two_sides_holding_a_login_of_one_provider_are_a_conflict_named_by_provider_only_in_both_directions() { From 8bcd6d67e9caa68aa615714bd755ab7ad549f15a Mon Sep 17 00:00:00 2001 From: Heimdall Date: Sat, 14 Nov 2026 13:28:09 +0000 Subject: [PATCH 2/5] AccountMergeSaga.SettleStuckAsync, the admin Merges page and the sweeper's proof prune (ruling merge-stuck-end-only-typed-id, merge-proof-bearers-memory-bound) Co-Authored-By: Claude Sonnet 5.5 --- .../Features/Membership/AccountMergeLedger.cs | 5 +- .../Features/Membership/AccountMergeSaga.cs | 63 ++++++++++++++- .../Features/Membership/MergeProofs.cs | 11 ++- src/Bifrost.Web/Pages/App/Merges/Index.cshtml | 70 +++++++++++++++++ .../Pages/App/Merges/Index.cshtml.cs | 77 ++++++++++++++++++- .../AccountMergeSettlePageTests.cs | 6 +- .../AccountMergeSettleTests.cs | 3 +- 7 files changed, 222 insertions(+), 13 deletions(-) diff --git a/src/Bifrost.Web/Features/Membership/AccountMergeLedger.cs b/src/Bifrost.Web/Features/Membership/AccountMergeLedger.cs index c8ab2b9..7c74bc3 100644 --- a/src/Bifrost.Web/Features/Membership/AccountMergeLedger.cs +++ b/src/Bifrost.Web/Features/Membership/AccountMergeLedger.cs @@ -737,8 +737,8 @@ public sealed class AccountMergeLedger( } /// -/// The one merge loop, every 30 seconds: the saga advances every due in-flight merge (confirm, replay, undo), then the ledger finalizes -/// merges past their grace. A new scope per tick, a warning on failure, the SealedBallotCloser shape. No second merge hosted service exists. +/// The one merge loop, every 30 seconds: the proof holder drops expired bearers, the saga advances every due in-flight merge (confirm, +/// replay, undo), then the ledger finalizes merges past their grace. A new scope per tick, a warning on failure, the SealedBallotCloser shape. No second merge hosted service exists. /// public sealed class AccountMergeSweeper(IServiceScopeFactory scopes, TimeProvider clock, ILogger logger) : BackgroundService { @@ -753,6 +753,7 @@ public sealed class AccountMergeSweeper(IServiceScopeFactory scopes, TimeProvide { using var scope = scopes.CreateScope(); var now = clock.GetUtcNow(); + scope.ServiceProvider.GetRequiredService().PruneExpired(); await scope.ServiceProvider.GetRequiredService().AdvanceDueAsync(now, stoppingToken); await scope.ServiceProvider.GetRequiredService().FinalizeDueAsync(now, stoppingToken); } diff --git a/src/Bifrost.Web/Features/Membership/AccountMergeSaga.cs b/src/Bifrost.Web/Features/Membership/AccountMergeSaga.cs index a7662ba..6e60db5 100644 --- a/src/Bifrost.Web/Features/Membership/AccountMergeSaga.cs +++ b/src/Bifrost.Web/Features/Membership/AccountMergeSaga.cs @@ -431,9 +431,66 @@ public sealed partial class AccountMergeSaga( // ---- operator settle ---- - public Task SettleStuckAsync( - Guid mergeId, SettleChoice choice, string typedMoveId, Guid operatorUserAccountId, CancellationToken cancellationToken) => - throw new NotImplementedException(); + /// The Heimdall answers no replay can change: the one the saga recorded and stopped on, the key ring's loss, and a move Heimdall holds but will not confirm. + public static readonly string[] SettleableOutcomes = [ReceiptMismatch, ProofsUnreadable, MoveIdConflict]; + + /// + /// Whether an operator may end this merge's leg: in flight (Confirmed or UndoRequested) and stopped on one of the closed + /// outcomes. Derived from the row on every read; there is no stuck flag. + /// + public static bool IsSettleable(AccountMerge merge) => + merge.State is AccountMergeState.Confirmed or AccountMergeState.UndoRequested && SettleableOutcomes.Contains(merge.LastOutcome); + + /// + /// A platform admin's end of a stuck leg (ruling merge-stuck-end-only-typed-id): FailForward turns a stuck Confirmed merge into + /// Failed, AbandonUndo returns a stuck UndoRequested merge to Committed with its grace. Only with the leg's stored move id typed back. + /// Never adopts or invents a Heimdall receipt; a merge Heimdall moved (HeimdallMoved, Committed) is never settleable. One save + /// guarded by State carries the transition and its audit row (choice only): the loser of a race with + /// answers Lost and changes nothing. The caller has checked the operator's authority. + /// + public async Task SettleStuckAsync( + Guid mergeId, SettleChoice choice, string typedMoveId, Guid operatorUserAccountId, CancellationToken cancellationToken) + { + var forward = choice == SettleChoice.FailForward; + var merge = await LoadAsync(mergeId, cancellationToken); + if (merge is null || merge.State != (forward ? AccountMergeState.Confirmed : AccountMergeState.UndoRequested) || !IsSettleable(merge)) + { + return new SettleOutcome.NotStuck(); + } + + if (!string.Equals(typedMoveId, forward ? merge.HeimdallMoveId : merge.HeimdallReverseMoveId, StringComparison.Ordinal)) + { + return new SettleOutcome.WrongMoveId(); + } + + merge.SealedProofs = null; + merge.NextAttemptUtc = null; + if (forward) + { + merge.State = AccountMergeState.Failed; + merge.FailedUtc = timeProvider.GetUtcNow(); + merge.FailureCode = OperatorFailed; + merge.LastOutcome = OperatorFailed; + } + else + { + merge.State = AccountMergeState.Committed; + merge.LastOutcome = UndoOperatorAbandoned; + } + + try + { + await audit.RecordAsync(operatorUserAccountId, nameof(AccountMerge), merge.Id, "merge.settled", choice.ToString(), cancellationToken); + } + catch (DbUpdateConcurrencyException) + { + dbContext.ChangeTracker.Clear(); + return new SettleOutcome.Lost(); + } + + logger.LogInformation("Account merge {MergeId} {State}: settled by an operator ({Choice})", merge.Id, merge.State, choice); + return new SettleOutcome.Settled(); + } // ---- helpers ---- diff --git a/src/Bifrost.Web/Features/Membership/MergeProofs.cs b/src/Bifrost.Web/Features/Membership/MergeProofs.cs index a9d614f..86b45f4 100644 --- a/src/Bifrost.Web/Features/Membership/MergeProofs.cs +++ b/src/Bifrost.Web/Features/Membership/MergeProofs.cs @@ -61,7 +61,7 @@ public abstract record UndoPeekResult /// the member proves again. It decides nothing about the merge: the saga owns consent, the move and the lifecycle, and Heimdall judges /// each proof's tier and freshness. is only a margin under Heimdall's 300 s weak window, so a stale proof /// is not sent to fail a merge. Provider is the only proof bookkeeping Bifrost has (the redeem names the provider, not the identity), -/// so a login list holding two logins of one provider is refused, and two sides holding a login of one provider are a conflict Peek names before consent. Every access drops expired bearers. Peek never consumes: the caller s once the saga +/// so a login list holding two logins of one provider is refused, and two sides holding a login of one provider are a conflict Peek names before consent. Every access, and the merge sweeper's tick, drops expired bearers. Peek never consumes: the caller s once the saga /// has taken the proofs, so a refusal that wrote nothing does not cost a re-proof. /// public sealed class MergeProofs(TimeProvider timeProvider) @@ -186,7 +186,14 @@ public sealed class MergeProofs(TimeProvider timeProvider) } } - public void PruneExpired() => throw new NotImplementedException(); + /// Drops every expired bearer without any member touching the page; the merge sweeper's tick calls it, so none outlives plus one tick. + public void PruneExpired() + { + lock (_gate) + { + Prune(timeProvider.GetUtcNow()); + } + } private static List UnprovedProviders(Side side, IEnumerable providers) => [.. providers.Distinct(StringComparer.Ordinal).Where(x => !side.Proofs.ContainsKey(x))]; diff --git a/src/Bifrost.Web/Pages/App/Merges/Index.cshtml b/src/Bifrost.Web/Pages/App/Merges/Index.cshtml index 861f53a..1f8d341 100644 --- a/src/Bifrost.Web/Pages/App/Merges/Index.cshtml +++ b/src/Bifrost.Web/Pages/App/Merges/Index.cshtml @@ -1,2 +1,72 @@ @page +@using Bifrost.Web.Domain @model Bifrost.Web.Pages.App.Merges.IndexModel +@{ + ViewData["Title"] = "Stuck merges"; +} + +
+
+ Merges +

Stuck account merges

+

+ A merge listed here has stopped advancing on its own. Before ending one, read Heimdall's identity move by the move id shown + (docs/account-merge.md, Stuck merges). Ending a merge cannot be undone, and Bifrost never adopts a move Heimdall made. +

+ + @if (!string.IsNullOrEmpty(Model.Notice)) + { + + } +
+
+ +
+ @if (Model.Merges.Count == 0) + { +
No merge is stuck.
+ } + else + { +
+ + + + + + + + + + + + + @foreach (var merge in Model.Merges) + { + + + + + + + + + } + +
MergeStateLast outcomeAttemptsMove id of this legEnd it
@merge.Id@merge.State@merge.LastOutcome@merge.Attempts@merge.MoveId + @if (!merge.CanSettle) + { + Still being retried; nothing to end. + } + else + { +
+ + + +
+ } +
+
+ } +
diff --git a/src/Bifrost.Web/Pages/App/Merges/Index.cshtml.cs b/src/Bifrost.Web/Pages/App/Merges/Index.cshtml.cs index 0506cb7..865cb42 100644 --- a/src/Bifrost.Web/Pages/App/Merges/Index.cshtml.cs +++ b/src/Bifrost.Web/Pages/App/Merges/Index.cshtml.cs @@ -1,9 +1,82 @@ +using Bifrost.Web.Data; +using Bifrost.Web.Domain; +using Bifrost.Web.Features.Membership; +using Bifrost.Web.Features.Shared; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc.RazorPages; +using Microsoft.EntityFrameworkCore; namespace Bifrost.Web.Pages.App.Merges; -public sealed class IndexModel : PageModel +/// One in-flight merge the sweeper has stopped advancing, as an operator may see it: no account, Heimdall id or login of either side. +public sealed record StuckMerge(Guid Id, AccountMergeState State, string LastOutcome, int Attempts, string MoveId, bool CanSettle); + +/// +/// The operator's door for a stuck merge (ruling merge-stuck-end-only-typed-id), platform admins only. It lists in-flight merges that +/// stopped on a settleable outcome or have been retried eight times, with the stored move id of the leg in flight, and passes an +/// admin's choice and the typed move id to , the one writer. It calls no Heimdall. +/// +public sealed class IndexModel(BifrostDbContext dbContext, ICurrentBifrostActorAccessor actorAccessor, AccountMergeSaga saga) : PageModel { - public IActionResult OnGet() => NotFound(); + private const int RetriesBeforeListed = 8; + + public const string Url = "/App/Merges"; + + private static readonly IReadOnlyDictionary ResultWords = new Dictionary + { + ["settled"] = "The merge was ended and the sweeper will not touch it again.", + ["not_stuck"] = "Nothing changed: that merge is not stuck on a state this choice can end.", + ["wrong_move_id"] = "Nothing changed: the move id typed is not the stored one of that leg.", + ["lost"] = "Nothing changed: the merge moved on by itself while you were settling it." + }; + + public IReadOnlyList Merges { get; private set; } = []; + + /// The words for the closed ?result= code of the last settle; null for no code or one this page does not know. + public string? Notice { get; private set; } + + public async Task OnGetAsync(string? result, CancellationToken cancellationToken) + { + if (!(await actorAccessor.GetAsync(cancellationToken)).IsPlatformAdmin) + { + return Forbid(); + } + + Notice = result is not null && ResultWords.TryGetValue(result, out var words) ? words : null; + var rows = await dbContext.AccountMerges.AsNoTracking() + .Where(x => (x.State == AccountMergeState.Confirmed || x.State == AccountMergeState.UndoRequested) + && (AccountMergeSaga.SettleableOutcomes.Contains(x.LastOutcome) || x.Attempts >= RetriesBeforeListed)) + .OrderBy(x => x.CreatedUtc) + .ToListAsync(cancellationToken); + Merges = [.. rows.Select(x => new StuckMerge( + x.Id, x.State, x.LastOutcome, x.Attempts, + (x.State == AccountMergeState.UndoRequested ? x.HeimdallReverseMoveId : x.HeimdallMoveId) ?? string.Empty, + AccountMergeSaga.IsSettleable(x)))]; + return Page(); + } + + public Task OnPostFailForwardAsync(Guid mergeId, string? moveId, CancellationToken cancellationToken) => + SettleAsync(mergeId, SettleChoice.FailForward, moveId, cancellationToken); + + public Task OnPostAbandonUndoAsync(Guid mergeId, string? moveId, CancellationToken cancellationToken) => + SettleAsync(mergeId, SettleChoice.AbandonUndo, moveId, cancellationToken); + + private async Task SettleAsync(Guid mergeId, SettleChoice choice, string? moveId, CancellationToken cancellationToken) + { + var actor = await actorAccessor.GetAsync(cancellationToken); + if (!actor.CanManageMembership || actor.UserAccount is null) + { + return Forbid(); + } + + var outcome = await saga.SettleStuckAsync(mergeId, choice, moveId ?? string.Empty, actor.UserAccount.Id, cancellationToken); + var code = outcome switch + { + SettleOutcome.Settled => "settled", + SettleOutcome.WrongMoveId => "wrong_move_id", + SettleOutcome.Lost => "lost", + _ => "not_stuck" + }; + return Redirect($"{Url}?result={code}"); + } } diff --git a/tests/Bifrost.Web.Tests/AccountMergeSettlePageTests.cs b/tests/Bifrost.Web.Tests/AccountMergeSettlePageTests.cs index 81e53ab..5337eb8 100644 --- a/tests/Bifrost.Web.Tests/AccountMergeSettlePageTests.cs +++ b/tests/Bifrost.Web.Tests/AccountMergeSettlePageTests.cs @@ -52,12 +52,12 @@ public sealed class AccountMergeSettlePageTests public Task<(HttpResponseMessage Response, string Html)> Get(string session, string query = "") => MergePage.GetAsync(Rig.Host, session, query, Url); public async Task Post(string session, string handler, string mergeId, string moveId, MergePage.Antiforgery? token = null) => - await MergePage.PostAsync(Rig.Host, session, token ?? await MergePage.TokenAsync(Rig.Host, session, MergePage.Url), handler, + await MergePage.PostAsync(Rig.Host, session, token ?? await MergePage.TokenAsync(Rig.Host, session, HeimdallHost.LinkedAccountsUrl), handler, new Dictionary { ["mergeId"] = mergeId, ["moveId"] = moveId }, Url); } private static bool Refused(HttpResponseMessage response) => - response.StatusCode == HttpStatusCode.Forbidden || (response.StatusCode == HttpStatusCode.Redirect && (response.Headers.Location?.OriginalString.Contains("AccessDenied", StringComparison.OrdinalIgnoreCase) ?? false)); + response.StatusCode == HttpStatusCode.Forbidden || (response.StatusCode == HttpStatusCode.Redirect && (response.Headers.Location?.OriginalString.Contains("/Membership/Status?ReturnUrl=", StringComparison.Ordinal) ?? false)); [Fact] public async Task A_member_who_is_not_a_platform_admin_is_refused_on_the_listing_and_on_both_posts_and_nothing_changes() @@ -67,7 +67,7 @@ public sealed class AccountMergeSettlePageTests var requests = w.Rig.Moves.Requests.Count; var (get, html) = await w.Get(w.MemberSession); - Assert.True(Refused(get), $"listing gave {(int)get.StatusCode}"); + Assert.True(Refused(get), $"listing gave {(int)get.StatusCode} {MergePage.Location(get)}"); Assert.DoesNotContain(merge.HeimdallMoveId!, html); Assert.True(Refused(await w.Post(w.MemberSession, "FailForward", w.Forward.Id.ToString(), merge.HeimdallMoveId!))); Assert.True(Refused(await w.Post(w.MemberSession, "AbandonUndo", w.Forward.Id.ToString(), merge.HeimdallMoveId!))); diff --git a/tests/Bifrost.Web.Tests/AccountMergeSettleTests.cs b/tests/Bifrost.Web.Tests/AccountMergeSettleTests.cs index 608d6a4..839cc60 100644 --- a/tests/Bifrost.Web.Tests/AccountMergeSettleTests.cs +++ b/tests/Bifrost.Web.Tests/AccountMergeSettleTests.cs @@ -257,7 +257,8 @@ public sealed class AccountMergeSettleTests Assert.IsType(outcome); Assert.Equal(AccountMergeState.Committed, (await rig.MergeAsync(stuck.Id)).State); Assert.Empty((await rig.MergeAsync(stuck.Id)).FailureCode); - Assert.Empty(await AuditAsync(rig, stuck.Id)); + // The audit row rides in the same SaveChanges as the transition, so a real database drops both; the in-memory provider + // applies entries one by one and cannot show that half. Assert.IsType(await rig.StandingAsync(stuck.Folded)); } From 11fc87c8e4b1af8cc14d9ac5c798193bd69713de Mon Sep 17 00:00:00 2001 From: Heimdall Date: Sat, 14 Nov 2026 13:28:45 +0000 Subject: [PATCH 3/5] Test: the provider-conflict words say to prove that account again after unlinking; an operator settle has its own status words (red) Co-Authored-By: Claude Sonnet 5.5 --- tests/Bifrost.Web.Tests/MergePreviewTests.cs | 11 +++++++++++ tests/Bifrost.Web.Tests/MergeWordingTests.cs | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/tests/Bifrost.Web.Tests/MergePreviewTests.cs b/tests/Bifrost.Web.Tests/MergePreviewTests.cs index 47e6b54..3e9d8c8 100644 --- a/tests/Bifrost.Web.Tests/MergePreviewTests.cs +++ b/tests/Bifrost.Web.Tests/MergePreviewTests.cs @@ -17,6 +17,17 @@ public sealed class MergePreviewTests private static MergePlan Plan(IEnumerable entries, IReadOnlyDictionary? left = null) => new([], [.. entries], left ?? new Dictionary()); + [Fact] + public void The_provider_conflict_names_the_providers_and_says_to_prove_that_side_again_after_unlinking() + { + // The conflict is read from the login list held with the proofs, which unlinking does not change until the member proves again. + var words = MergePreview.ProviderConflict(["Discord", "Twitch"]); + + Assert.Contains("Discord and Twitch", words); + Assert.Contains("Unlink", words); + Assert.Contains("prove that account again", words); + } + [Fact] public void Every_block_code_has_its_own_words_and_none_names_an_account() { diff --git a/tests/Bifrost.Web.Tests/MergeWordingTests.cs b/tests/Bifrost.Web.Tests/MergeWordingTests.cs index 7f1a147..6f63e89 100644 --- a/tests/Bifrost.Web.Tests/MergeWordingTests.cs +++ b/tests/Bifrost.Web.Tests/MergeWordingTests.cs @@ -43,6 +43,17 @@ public sealed class MergeWordingTests [InlineData("malformed or mismatched receipt")] public void Every_outcome_the_saga_stores_has_words(string outcome) => Assert.False(string.IsNullOrWhiteSpace(MergeWording.Outcome(outcome)), outcome); + [Theory] + [InlineData("operator_failed", "ended")] + [InlineData("undo_operator_abandoned", "merge stands")] + public void An_operators_settle_has_its_own_words_not_the_retry_line(string outcome, string says) + { + var words = MergeWording.Outcome(outcome); + + Assert.Contains(says, words); + Assert.DoesNotContain("try again", words); + } + [Theory] [InlineData("HTTP 502")] [InlineData("HEIMDALL FREE TEXT")] From 870b475795f2733b7bdd014707dea2495899a8a3 Mon Sep 17 00:00:00 2001 From: Heimdall Date: Sat, 14 Nov 2026 13:29:56 +0000 Subject: [PATCH 4/5] Provider-conflict words say to prove that account again; status words for an operator settle; docs: Stuck merges, the sweeper's prune Co-Authored-By: Claude Sonnet 5.5 --- docs/account-merge.md | 19 +++++++++++++++++-- .../Features/Membership/AccountMergeLedger.cs | 1 + .../Features/Membership/MergePreview.cs | 2 +- .../Features/Membership/MergeWording.cs | 2 ++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/docs/account-merge.md b/docs/account-merge.md index 41a4c1f..75665ba 100644 --- a/docs/account-merge.md +++ b/docs/account-merge.md @@ -62,11 +62,26 @@ Left untouched (counted by the plan): consent receipts, passkeys, ledger entries `ConfirmAsync` refuses (writing nothing) a third-party actor (`NotParty`), a plan with blocks (`Blocked`, codes only), a plan other than the one the member saw (`PreviewStale`: `MergePlan.Digest(survivorId, foldedId)`, a SHA-256 over a length-prefixed canonical text of the plan), and any proof of another Heimdall account (`ProofAccountMismatch`). A second live merge folding the same account, or a chain, is refused by the database (the live-merge unique index and the chain trigger) and answered `Blocked(merge_in_flight)`. Confirm and the undo request run through `AccountMergeLedger.WriteAsAsync` for the acting member, like every writer that acts for an account. -**Heimdall down.** `Unknown` (a timeout, a 5xx, a 429, an unrecognised 4xx, `move_id_conflict`) never fails a merge: the saga counts `Attempts`, sets `NextAttemptUtc` (15 s, 1 min, 5 min, then 15 min) and replays the same stored id; the folded account stays Frozen until Heimdall answers. Heimdall's answer to a replay of a move it already stored does not depend on the proofs' age, but a move that never reached Heimdall does: an outage longer than the proofs' freshness (300 s for a login's weak proof, 600 s for a passkey's strong one) ends, when Heimdall is back, in `Refused proof_stale` and a Failed merge; the member confirms again with fresh sign-ins. A replay whose proofs no longer verify (Heimdall's key regenerated) answers `move_id_conflict`, stays Unknown and can never settle: that is follow-up `merge-stuck-operator-path`, as is a receipt that names an account other than the destination (`receipt_mismatch`: proofs wiped, no retry) and a sealed payload the key ring can no longer open (`proofs_unreadable`: retried in an hour, state unchanged, because Heimdall may already have moved). +**Heimdall down.** `Unknown` (a timeout, a 5xx, a 429, an unrecognised 4xx, `move_id_conflict`) never fails a merge: the saga counts `Attempts`, sets `NextAttemptUtc` (15 s, 1 min, 5 min, then 15 min) and replays the same stored id; the folded account stays Frozen until Heimdall answers. Heimdall's answer to a replay of a move it already stored does not depend on the proofs' age, but a move that never reached Heimdall does: an outage longer than the proofs' freshness (300 s for a login's weak proof, 600 s for a passkey's strong one) ends, when Heimdall is back, in `Refused proof_stale` and a Failed merge; the member confirms again with fresh sign-ins. A replay whose proofs no longer verify (Heimdall's key regenerated) answers `move_id_conflict` and stays Unknown; a receipt that names an account other than the destination is `receipt_mismatch` (proofs wiped, no retry); a sealed payload the key ring can no longer open is `proofs_unreadable` (retried in an hour, state unchanged, because Heimdall may already have moved). None of the three ends by itself: see Stuck merges. **Sealed proofs.** The saga holds the two Heimdall bearers (`ruling merge-sealed-proofs-until-move-final`) only between Confirmed (or UndoRequested) and Heimdall's final answer: protected with Data Protection (purpose `Bifrost.AccountMerge.Proofs.v1`), never logged, wiped in the save that records Moved or Refused. Logs name the merge id, state, outcome class and refusal code; never a token, account id, identity id or provider user id. Proof coverage is Heimdall's rule (`move-proof-strict-passkey`): a proof through one login moves only that login, and only an asserted passkey speaks for the whole account, so a folded account's every login must be proved (or a passkey asserted) or Heimdall answers `identity_not_proved`. A `moveId` replayed with other identities returns the first receipt (`move-replay-first-receipt`). -`AccountMergeSweeper` is the one merge loop (30 s): per tick, in one scope, `AccountMergeSaga.AdvanceDueAsync` (every in-flight merge whose `NextAttemptUtc` is null or due, oldest first, a failure logged and skipped), then `FinalizeDueAsync`. +`AccountMergeSweeper` is the one merge loop (30 s): per tick, in one scope, `MergeProofs.PruneExpired` (a proof bearer lives only in process memory and is dropped 270 s after its proof, on access and by this tick, so none outlives that plus one tick whether or not anyone opens the page; `heimdall-body:ruling:merge-proof-bearers-memory-bound`), then `AccountMergeSaga.AdvanceDueAsync` (every in-flight merge whose `NextAttemptUtc` is null or due, oldest first, a failure logged and skipped), then `FinalizeDueAsync`. + +## Stuck merges + +`/App/Merges` (platform admins only; the `/App` folder's ActiveMember policy, then `IsPlatformAdmin`) lists in-flight merges (Confirmed or UndoRequested) that stopped on `receipt_mismatch`, `proofs_unreadable` or `move_id_conflict`, or were retried eight times, with the merge id, state, last outcome, attempts and the stored move id of the leg in flight. It shows no Heimdall account id, login or account name, and calls no Heimdall. Only the first three outcomes can be ended (`AccountMergeSaga.SettleableOutcomes`); a merge listed for its attempts alone is still being retried. + +Two actions (`heimdall-body:ruling:merge-stuck-end-only-typed-id`), both `AccountMergeSaga.SettleStuckAsync`, both needing the leg's stored move id typed back (the forward id for a Confirmed merge, the reverse id for an UndoRequested one) and both writing one `merge.settled` audit row that names the merge and the choice only: + +| Action | Applies to | Result | +| --- | --- | --- | +| Fail the merge (`FailForward`) | Confirmed, stuck | `Failed`, `FailureCode` and `LastOutcome` `operator_failed`, proofs wiped. The folded account is free again; the pair can be confirmed anew. | +| Give up the undo (`AbandonUndo`) | UndoRequested, stuck | back to `Committed`, `LastOutcome` `undo_operator_abandoned`, proofs wiped, the grace as it was. The merge stands; the logins stay with the survivor. | + +HeimdallMoved and Committed are never settleable: Heimdall answered Moved with a matching receipt there, and the saga rolls forward. A settle and the sweeper's `AdvanceAsync` meet at the `State` concurrency token; the loser changes nothing and the other transition stands (answered `lost`). + +Before ending a merge, read Heimdall's `identity_moves` row by the stored move id. If Heimdall did move the logins, failing the merge leaves them moved with no Bifrost merge to match: repair or reverse at Heimdall first. For `receipt_mismatch`, repair Heimdall's or Bifrost's account link before failing. Adopting a move Heimdall made (writing HeimdallMoved from a receipt) stays manual SQL until follow-up `merge-operator-adopt-moved`; the page never does it. ## Pages and flags diff --git a/src/Bifrost.Web/Features/Membership/AccountMergeLedger.cs b/src/Bifrost.Web/Features/Membership/AccountMergeLedger.cs index 7c74bc3..24a61ef 100644 --- a/src/Bifrost.Web/Features/Membership/AccountMergeLedger.cs +++ b/src/Bifrost.Web/Features/Membership/AccountMergeLedger.cs @@ -44,6 +44,7 @@ public sealed record MergePlan( Field(entry.Kind.ToString()); Field(entry.Action.ToString()); // A Created row does not exist yet: its id is minted anew by every plan, so it cannot be part of what the member saw. + // Left unpinned on purpose: the mutant that keeps the id in (digest-includes-created-id in tools/merge-saga-mutants.py) is behaviourally equivalent bar one benign case. Field(entry.Action == AccountMergeEntryAction.Created ? null : entry.RecordId.ToString("N")); Field(entry.PriorUserAccountId?.ToString("N")); Field(entry.PriorText); diff --git a/src/Bifrost.Web/Features/Membership/MergePreview.cs b/src/Bifrost.Web/Features/Membership/MergePreview.cs index 02811ee..d241a68 100644 --- a/src/Bifrost.Web/Features/Membership/MergePreview.cs +++ b/src/Bifrost.Web/Features/Membership/MergePreview.cs @@ -67,7 +67,7 @@ public static class MergePreview /// The block for two accounts that both hold a login of one provider, which Heimdall will not combine. Providers only: no account is named. public static string ProviderConflict(IEnumerable providerNames) => - $"Both accounts hold a {string.Join(" and ", providerNames)} login, and Heimdall will not combine two logins of one provider. Unlink one of them first."; + $"Both accounts hold a {string.Join(" and ", providerNames)} login, and Heimdall will not combine two logins of one provider. Unlink one of them first, then prove that account again: the sign-in you gave still carries the old list of its logins."; /// What every merge says, whatever the plan: what it cannot carry over, what it asks again, and how long it can be undone. public static IReadOnlyList Statements { get; } = diff --git a/src/Bifrost.Web/Features/Membership/MergeWording.cs b/src/Bifrost.Web/Features/Membership/MergeWording.cs index f51e060..9d6984d 100644 --- a/src/Bifrost.Web/Features/Membership/MergeWording.cs +++ b/src/Bifrost.Web/Features/Membership/MergeWording.cs @@ -47,6 +47,8 @@ public static class MergeWording "moved" => "Heimdall moved the logins.", "refused" => "Heimdall refused the merge.", "undo_requested" => "Waiting for Heimdall to give the logins back.", + "operator_failed" => "A steward ended this merge. Nothing was merged.", + "undo_operator_abandoned" => "A steward ended the undo, so the merge stands.", "receipt_mismatch" => "Heimdall's answer did not match the merge. An operator must look at it.", "proofs_unreadable" => "The stored proofs cannot be read. An operator must look at it.", "transport" or "move_id_conflict" or "malformed or mismatched receipt" => "Heimdall could not be reached or gave an answer Bifrost could not use. It will try again.", From 6e705852fbbfbc9ff33b96be4cdb255e7a251992 Mon Sep 17 00:00:00 2001 From: Heimdall Date: Sat, 14 Nov 2026 14:00:59 +0000 Subject: [PATCH 5/5] Settle: live-PG pins (lost race writes no audit row, audit fault rolls back, concurrent settles once); SettleableOutcomes read-only; docs: a settle does not fence an in-flight replay Co-Authored-By: Claude Sonnet 5.5 --- docs/account-merge.md | 2 +- .../Features/Membership/AccountMergeSaga.cs | 2 +- .../AccountMergePostgresTests.cs | 83 +++++++++++++++++++ 3 files changed, 85 insertions(+), 2 deletions(-) diff --git a/docs/account-merge.md b/docs/account-merge.md index 75665ba..67e224b 100644 --- a/docs/account-merge.md +++ b/docs/account-merge.md @@ -79,7 +79,7 @@ Two actions (`heimdall-body:ruling:merge-stuck-end-only-typed-id`), both `Accoun | Fail the merge (`FailForward`) | Confirmed, stuck | `Failed`, `FailureCode` and `LastOutcome` `operator_failed`, proofs wiped. The folded account is free again; the pair can be confirmed anew. | | Give up the undo (`AbandonUndo`) | UndoRequested, stuck | back to `Committed`, `LastOutcome` `undo_operator_abandoned`, proofs wiped, the grace as it was. The merge stands; the logins stay with the survivor. | -HeimdallMoved and Committed are never settleable: Heimdall answered Moved with a matching receipt there, and the saga rolls forward. A settle and the sweeper's `AdvanceAsync` meet at the `State` concurrency token; the loser changes nothing and the other transition stands (answered `lost`). +HeimdallMoved and Committed are never settleable: Heimdall answered Moved with a matching receipt there, and the saga rolls forward. A settle and the sweeper's `AdvanceAsync` meet at the `State` concurrency token; the loser changes nothing and the other transition stands (answered `lost`). A settle does not fence a replay already in flight to Heimdall, so read `identity_moves` first and read it again after. Before ending a merge, read Heimdall's `identity_moves` row by the stored move id. If Heimdall did move the logins, failing the merge leaves them moved with no Bifrost merge to match: repair or reverse at Heimdall first. For `receipt_mismatch`, repair Heimdall's or Bifrost's account link before failing. Adopting a move Heimdall made (writing HeimdallMoved from a receipt) stays manual SQL until follow-up `merge-operator-adopt-moved`; the page never does it. diff --git a/src/Bifrost.Web/Features/Membership/AccountMergeSaga.cs b/src/Bifrost.Web/Features/Membership/AccountMergeSaga.cs index 6e60db5..153ee80 100644 --- a/src/Bifrost.Web/Features/Membership/AccountMergeSaga.cs +++ b/src/Bifrost.Web/Features/Membership/AccountMergeSaga.cs @@ -432,7 +432,7 @@ public sealed partial class AccountMergeSaga( // ---- operator settle ---- /// The Heimdall answers no replay can change: the one the saga recorded and stopped on, the key ring's loss, and a move Heimdall holds but will not confirm. - public static readonly string[] SettleableOutcomes = [ReceiptMismatch, ProofsUnreadable, MoveIdConflict]; + public static readonly IReadOnlyList SettleableOutcomes = [ReceiptMismatch, ProofsUnreadable, MoveIdConflict]; /// /// Whether an operator may end this merge's leg: in flight (Confirmed or UndoRequested) and stopped on one of the closed diff --git a/tests/Bifrost.Web.Tests/AccountMergePostgresTests.cs b/tests/Bifrost.Web.Tests/AccountMergePostgresTests.cs index 06981a6..e353e0f 100644 --- a/tests/Bifrost.Web.Tests/AccountMergePostgresTests.cs +++ b/tests/Bifrost.Web.Tests/AccountMergePostgresTests.cs @@ -865,6 +865,89 @@ public sealed class AccountMergePostgresTests(MergePostgres postgres) : IClassFi Assert.True(blocked); } + /// A Confirmed merge with sealed proofs, stopped on the given outcome, and the operator who will settle it. + private async Task<(AccountMergeLedgerTests.World World, Heimdall Heimdall, Guid Id, string MoveId, Guid Operator)> StuckConfirmedAsync(string outcome) + { + var world = await postgres.NewWorldAsync(); + var survivor = await world.AccountAsync("survivor"); + var folded = await world.AccountAsync("folded"); + var operatorId = (await world.AccountAsync("operator")).Id; + var heimdall = new Heimdall(world); + Guid id; + await using (var db = world.Db()) + { + var digest = (await world.Ledger(db).PlanAsync(survivor.Id, folded.Id, default)).Digest(survivor.Id, folded.Id); + id = Assert.IsType(await heimdall.Saga(world, db).ConfirmAsync( + new ConfirmMergeRequest(survivor.Id, survivor.Id, folded.Id, digest, heimdall.Proof(survivor, "twitch"), [heimdall.Proof(folded, "discord")]), default)).MergeId; + } + + await ExecuteAsync(world, $"UPDATE \"AccountMerges\" SET \"State\" = 'Confirmed', \"LastOutcome\" = '{outcome}' WHERE \"Id\" = '{id}'"); + await using var read = world.Db(); + var merge = await read.AccountMerges.AsNoTracking().SingleAsync(); + Assert.NotNull(merge.SealedProofs); + return (world, heimdall, id, merge.HeimdallMoveId!, operatorId); + } + + [PostgresFact] + public async Task A_settle_that_loses_to_a_rival_state_write_inside_its_save_window_writes_nothing_and_no_audit_row() + { + var (world, heimdall, id, moveId, operatorId) = await StuckConfirmedAsync("move_id_conflict"); + var settler = AccountMergeLedgerTests.World.OnPostgres(ConnectionOf(world), new BeforeFirstSave(() => + ExecuteAsync(world, $"UPDATE \"AccountMerges\" SET \"State\" = 'HeimdallMoved' WHERE \"Id\" = '{id}'"))); + await using (var db = settler.Db()) + { + Assert.IsType(await heimdall.Saga(settler, db).SettleStuckAsync(id, SettleChoice.FailForward, moveId, operatorId, default)); + } + + await using var read = world.Db(); + var merge = await read.AccountMerges.AsNoTracking().SingleAsync(); + Assert.Equal(AccountMergeState.HeimdallMoved, merge.State); + Assert.NotNull(merge.SealedProofs); + Assert.Equal("move_id_conflict", merge.LastOutcome); + Assert.Equal(0, await read.AuditEvents.CountAsync()); + } + + [PostgresFact] + public async Task A_settle_whose_audit_insert_fails_rolls_the_transition_back() + { + var (world, heimdall, id, moveId, operatorId) = await StuckConfirmedAsync("receipt_mismatch"); + await ExecuteAsync(world, """ + CREATE FUNCTION fail_audit() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected audit fault'; END $$; + CREATE TRIGGER fail_audit BEFORE INSERT ON "AuditEvents" FOR EACH ROW EXECUTE FUNCTION fail_audit(); + """); + await using (var db = world.Db()) + { + await Assert.ThrowsAsync(() => heimdall.Saga(world, db).SettleStuckAsync(id, SettleChoice.FailForward, moveId, operatorId, default)); + } + + await using var read = world.Db(); + var merge = await read.AccountMerges.AsNoTracking().SingleAsync(); + Assert.Equal(AccountMergeState.Confirmed, merge.State); + Assert.NotNull(merge.SealedProofs); + Assert.Equal("receipt_mismatch", merge.LastOutcome); + Assert.Equal(0, await read.AuditEvents.CountAsync()); + } + + [PostgresFact] + public async Task Two_concurrent_settles_of_one_stuck_merge_settle_once_lose_once_and_audit_once() + { + var (world, heimdall, id, moveId, operatorId) = await StuckConfirmedAsync("proofs_unreadable"); + var racers = AccountMergeLedgerTests.World.OnPostgres(ConnectionOf(world), new FirstSaveRendezvous()); + async Task Settle() + { + await using var db = racers.Db(); + return await heimdall.Saga(racers, db).SettleStuckAsync(id, SettleChoice.FailForward, moveId, operatorId, default); + } + + var both = await Task.WhenAll(Task.Run(Settle), Task.Run(Settle)); + + Assert.Single(both.OfType()); + Assert.Single(both.OfType()); + await using var read = world.Db(); + Assert.Equal(1, await read.AuditEvents.CountAsync()); + Assert.Equal(AccountMergeState.Failed, (await read.AccountMerges.AsNoTracking().SingleAsync()).State); + } + [PostgresFact] public async Task The_saga_migration_adds_and_drops_its_columns_and_keys_and_its_Down_is_refused_while_any_merge_exists() {