From 6950da865e12e7f3703cf7915ef3f2e153184f83 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 01:37:35 +0100 Subject: [PATCH] estate harness: every drive asserts the outcome it shows and run.sh fails when one does not Co-Authored-By: Claude Sonnet 5.5 --- harness/estate/check.cjs | 18 +++++++++++++++++ harness/estate/drive-failures.cjs | 33 +++++++++++++++++++++---------- harness/estate/drive-routes.cjs | 20 ++++++++++++++----- harness/estate/drive.cjs | 14 ++++++++++--- harness/estate/run.sh | 9 +++++---- 5 files changed, 72 insertions(+), 22 deletions(-) create mode 100644 harness/estate/check.cjs diff --git a/harness/estate/check.cjs b/harness/estate/check.cjs new file mode 100644 index 00000000..172fa301 --- /dev/null +++ b/harness/estate/check.cjs @@ -0,0 +1,18 @@ +// Assertions for the estate drives: every drive states the outcome it exists to show and exits +// non-zero when the page or the fake homeserver's log says otherwise. +const fs = require('fs'); +const fails = []; +exports.expect = (label, ok, detail = '') => { + console.log(`[ASSERT] ${ok ? 'PASS' : 'FAIL'} ${label}${ok ? '' : ` :: ${detail}`}`); + if (!ok) fails.push(label); +}; +/** Exit code for the drive: 4 when any assertion failed. */ +exports.done = () => (fails.length ? 4 : 0); +/** The fake homeserver's request log (HSLOG is shared with the server process). */ +exports.hsLog = () => fs.readFileSync(process.env.HSLOG, 'utf8').split('\n'); +/** Bodies of the account-data PUTs for m.direct, in order. */ +exports.mDirectPuts = () => + exports.hsLog().filter((l) => / PUT .*m\.direct BODY /.test(l)).map((l) => l.split(' BODY ')[1]); +/** Join requests the server saw for a room id. */ +exports.joinsOf = (roomId) => exports.hsLog().filter((l) => new RegExp(` JOIN ${roomId}$`).test(l)).length; +exports.isDm = (tier) => /^direct messages$/i.test(tier); diff --git a/harness/estate/drive-failures.cjs b/harness/estate/drive-failures.cjs index 9c81424a..1172c853 100644 --- a/harness/estate/drive-failures.cjs +++ b/harness/estate/drive-failures.cjs @@ -1,6 +1,7 @@ // Failure-mode scenarios for the estate join paths (estate tap and inbox Accept), run by run.sh // against fakehs.cjs with the env flag each mode needs. const { chromium } = require('playwright'); +const { expect, done, isDm, mDirectPuts, joinsOf } = require('./check.cjs'); const MODE = process.env.MODE; const T0 = Date.now(); const t = () => `+${Date.now() - T0}ms`; const log = (...a) => console.log(`[${MODE}]`, ...a); @@ -21,7 +22,8 @@ setTimeout(() => { console.log('DRIVER_TIMEOUT'); quit(3); }, 240000); const reveal = async (name) => { const loc = pg.locator(`[aria-label="Join ${name}"]`); if (await loc.count()) return loc; for (let r = 0; r < 2; r++) for (const h of HEADERS) { const hl = pg.getByText(h, { exact: true }); if (await hl.count()) { await hl.first().click(); await pg.waitForTimeout(250); if (await loc.count()) return loc; } } return loc; }; const tierOf = (name) => pg.evaluate((n) => { const s = document.body.innerText.split('\n').map((x) => x.trim()); const i = s.indexOf(n); if (i < 0) return 'absent'; for (let j = i; j >= 0; j--) if (/^(elsewhere|direct messages)$/i.test(s[j])) return s[j]; return 'none-above'; }, name); const tap = async (n) => { const l = await reveal(n); log('JOINCTRL', n, await l.count()); await l.first().click(); log('CLICK', n, t()); }; - const tiers = async (tag) => { await pg.goto('http://127.0.0.1:8080/#/home/'); await pg.waitForTimeout(2500); for (const h of HEADERS) { const hl = pg.getByText(h, { exact: true }); if (await hl.count()) { await hl.first().click(); await pg.waitForTimeout(250); } } log('TIER', tag, '| Friend DM', await tierOf('Friend DM'), '| Slow DM', await tierOf('Slow DM')); }; + const tiers = async (tag) => { await pg.goto('http://127.0.0.1:8080/#/home/'); await pg.waitForTimeout(2500); for (const h of HEADERS) { const hl = pg.getByText(h, { exact: true }); if (await hl.count()) { await hl.first().click(); await pg.waitForTimeout(250); } } const r = { friend: await tierOf('Friend DM'), slow: await tierOf('Slow DM') }; log('TIER', tag, '| Friend DM', r.friend, '| Slow DM', r.slow); return r; }; + const neverUnfiled = () => mDirectPuts().every((b) => b.includes('!dm1:test')); const acceptInInbox = async (name) => { // reach the inbox through the navigator's own Inbox tab await pg.goto('http://127.0.0.1:8080/#/home/'); await pg.waitForTimeout(2000); @@ -38,23 +40,34 @@ setTimeout(() => { console.log('DRIVER_TIMEOUT'); quit(3); }, 240000); }; if (MODE === 'pjfail') { await tap('Friend DM'); await pg.waitForTimeout(3000); - const bb = await body(); const k = bb.search(/Internal server error|Could not join|Joined\./); log('AFTER_FAIL', k >= 0 ? bb.slice(Math.max(0, k - 40), k + 80) : 'no message'); - await tap('Friend DM'); await pg.waitForTimeout(6000); await tiers('after retry'); + const bb = await body(); const k = bb.search(/Internal server error/); log('AFTER_FAIL', k >= 0 ? bb.slice(Math.max(0, k - 40), k + 80) : 'no message'); + expect('a failed m.direct write shows the refusal', k >= 0, bb.slice(0, 200)); + expect('no join is sent while the filing is failing', joinsOf('!dm1:test') === 0, String(joinsOf('!dm1:test'))); + await tap('Friend DM'); await pg.waitForTimeout(6000); const r = await tiers('after retry'); + expect('the retry joins once', joinsOf('!dm1:test') === 1, String(joinsOf('!dm1:test'))); + expect('Friend DM is filed under Direct Messages after the retry', isDm(r.friend), r.friend); } else if (MODE === 'jdrop') { await tap('Friend DM'); await pg.waitForTimeout(4000); - const bb = await body(); const k = bb.search(/fetch|Failed|network|Could not join|Joined\./i); log('AFTER_DROP', k >= 0 ? bb.slice(Math.max(0, k - 60), k + 80) : 'no message'); - await pg.waitForTimeout(6000); await tiers('after drop'); await open(); await tiers('after reload'); + const bb = await body(); const k = bb.search(/tap again to retry/i); log('AFTER_DROP', k >= 0 ? bb.slice(Math.max(0, k - 60), k + 80) : 'no message'); + expect('a dropped join shows the retryable message', k >= 0, bb.slice(0, 200)); + await pg.waitForTimeout(6000); const r1 = await tiers('after drop'); await open(); const r2 = await tiers('after reload'); + expect('the entry is kept after the drop', isDm(r1.friend) && neverUnfiled(), r1.friend); + expect('Friend DM is under Direct Messages after the reload', isDm(r2.friend), r2.friend); } else if (MODE === 'admin') { - await tap('Friend DM'); await pg.waitForTimeout(6000); await tiers('admin estate'); + await tap('Friend DM'); await pg.waitForTimeout(6000); const r = await tiers('admin estate'); + expect('an admin-invited DM is filed under Direct Messages', isDm(r.friend), r.friend); } else if (MODE === 'inboxadmin') { - await acceptInInbox('Friend DM'); await pg.waitForTimeout(6000); await tiers('admin inbox'); + await acceptInInbox('Friend DM'); await pg.waitForTimeout(6000); const r = await tiers('admin inbox'); + expect('an admin-invited DM accepted in the inbox is filed under Direct Messages', isDm(r.friend), r.friend); } else if (MODE === 'inboxreload') { await acceptInInbox('Slow DM'); await pg.waitForTimeout(2000); log('RELOAD', t()); await open('#/inbox/invites/'); - await pg.waitForTimeout(30000); await tiers('inbox reload'); + await pg.waitForTimeout(30000); const r = await tiers('inbox reload'); + expect('a DM accepted in the inbox is filed under Direct Messages after a reload', isDm(r.slow), r.slow); } else if (MODE === 'pjreload') { await tap('Slow DM'); await pg.waitForTimeout(2000); log('RELOAD', t()); await open(); await pg.waitForTimeout(8000); await tiers('pjreload'); - const l = pg.locator('[aria-label="Join Slow DM"]'); log('SLOW_DM_STILL_JOINABLE', await l.count()); + const l = pg.locator('[aria-label="Join Slow DM"]'); const n = await l.count(); log('SLOW_DM_STILL_JOINABLE', n); + expect('a join cut off by a reload stays joinable', n === 1, String(n)); } - await quit(0); + await quit(done()); })().catch((e) => { log('ERROR', String(e.stack || e).slice(0, 300)); quit(1); }); diff --git a/harness/estate/drive-routes.cjs b/harness/estate/drive-routes.cjs index 05e89df1..5def6635 100644 --- a/harness/estate/drive-routes.cjs +++ b/harness/estate/drive-routes.cjs @@ -1,4 +1,5 @@ const { chromium } = require('playwright'); +const { expect, done, isDm, mDirectPuts, joinsOf, hsLog } = require('./check.cjs'); const MODE = process.env.MODE; const T0 = Date.now(); const t = () => `+${Date.now() - T0}ms`; const log = (...a) => console.log(`[${MODE}]`, ...a); const HEADERS = ['GameCult', 'Projects', 'nibu', 'eureka', 'Nibu', 'Eureka', 'Elsewhere', 'Direct Messages']; @@ -13,21 +14,30 @@ setTimeout(() => { console.log('DRIVER_TIMEOUT'); quit(3); }, 120000); await pg.goto('http://127.0.0.1:8080/#/home/'); await pg.reload(); await pg.waitForTimeout(9000); const body = async () => (await pg.evaluate(() => document.body.innerText)).replace(/\s+/g, ' '); const tierOf = (n) => pg.evaluate((n) => { const s = document.body.innerText.split('\n').map((x) => x.trim()); const i = s.indexOf(n); if (i < 0) return 'absent'; for (let j = i; j >= 0; j--) if (/^(elsewhere|direct messages)$/i.test(s[j])) return s[j]; return 'none-above'; }, n); - const tiers = async (tag) => { await pg.goto('http://127.0.0.1:8080/#/home/'); await pg.waitForTimeout(2500); for (const h of HEADERS) { const hl = pg.getByText(h, { exact: true }); if (await hl.count()) { await hl.first().click(); await pg.waitForTimeout(250); } } log('TIER', tag, '| Friend DM', await tierOf('Friend DM')); }; + const tiers = async (tag) => { await pg.goto('http://127.0.0.1:8080/#/home/'); await pg.waitForTimeout(2500); for (const h of HEADERS) { const hl = pg.getByText(h, { exact: true }); if (await hl.count()) { await hl.first().click(); await pg.waitForTimeout(250); } } const f = await tierOf('Friend DM'); log('TIER', tag, '| Friend DM', f); return f; }; + let joined = false; if (MODE === 'deeplink') { for (const route of ['direct', 'home']) { await pg.goto(`http://127.0.0.1:8080/#/${route}/${encodeURIComponent('!dm1:test')}/`); await pg.waitForTimeout(4000); const bb = await body(); log('ROUTE', route, JSON.stringify(bb.slice(0, 300))); const j = pg.getByRole('button', { name: /^Join$/ }); log('JOIN_BUTTONS', route, await j.count()); - if (await j.count()) { await j.first().click(); log('CLICK_JOIN', route, t()); await pg.waitForTimeout(6000); break; } + if (await j.count()) { joined = true; await j.first().click(); log('CLICK_JOIN', route, t()); await pg.waitForTimeout(6000); break; } } - await tiers('after deeplink join'); + const f = await tiers('after deeplink join'); + const stamps = (re) => hsLog().filter((l) => re.test(l)).map((l) => Number(l.split(' ')[0])); + const put = stamps(/ PUT .*m\.direct BODY .*!dm1:test/)[0], join = stamps(/ JOIN !dm1:test$/)[0]; + expect('the deep link offers one Join button', joined, 'no Join button'); + expect('the DM is filed before the join is sent', put !== undefined && join !== undefined && put <= join, `put ${put} join ${join}`); + expect('the DM is filed under Direct Messages', isDm(f), f); } else if (MODE === 'gw504') { const HL = HEADERS; let l = pg.locator('[aria-label="Join Friend DM"]'); for (let r = 0; r < 2 && !(await l.count()); r++) for (const h of HL) { const hl = pg.getByText(h, { exact: true }); if (await hl.count()) { await hl.first().click(); await pg.waitForTimeout(250); } } log('JOINCTRL', await l.count()); await l.first().click(); log('CLICK', t()); await pg.waitForTimeout(4000); const bb = await body(); const k = bb.search(/504|Gateway|connection dropped|Could not join/i); log('AFTER_504', k >= 0 ? bb.slice(Math.max(0, k - 60), k + 120) : 'no message'); - await pg.waitForTimeout(4000); await tiers('after 504 (join delivered by sync)'); + expect('a gateway timeout shows the retryable message', /tap again to retry/i.test(bb), bb.slice(0, 200)); + await pg.waitForTimeout(4000); const f = await tiers('after 504 (join delivered by sync)'); + expect('the DM stays filed after a gateway timeout', mDirectPuts().every((b) => b.includes('!dm1:test')) && mDirectPuts().length > 0, JSON.stringify(mDirectPuts())); + expect('the DM is filed under Direct Messages once sync delivers it', isDm(f), f); } - await quit(0); + await quit(done()); })().catch((e) => { log('ERROR', String(e.stack || e).slice(0, 300)); quit(1); }); diff --git a/harness/estate/drive.cjs b/harness/estate/drive.cjs index ed9950ce..cf794323 100644 --- a/harness/estate/drive.cjs +++ b/harness/estate/drive.cjs @@ -1,8 +1,10 @@ const { chromium } = require('playwright'); +const { expect, done, isDm } = require('./check.cjs'); const MODE = process.env.MODE; // twodm | reload const T0 = Date.now(); const t = () => `+${Date.now() - T0}ms`; const log = (...a) => console.log(`[${MODE}]`, ...a); const HEADERS = ['GameCult', 'Projects', 'nibu', 'eureka', 'Nibu', 'Eureka', 'Elsewhere', 'Direct Messages']; +let tierBefore; (async () => { const b = await chromium.launch(); const ctx = await b.newContext({ viewport: { width: 390, height: 844 }, isMobile: true, hasTouch: true, deviceScaleFactor: 1 }); @@ -20,7 +22,7 @@ const HEADERS = ['GameCult', 'Projects', 'nibu', 'eureka', 'Nibu', 'Eureka', 'El await pg.waitForTimeout(45000); } else if (MODE === 'convert') { await tap('Friend DM'); await pg.waitForTimeout(6000); - log('TIER before', await tierOf('Friend DM')); + tierBefore = await tierOf('Friend DM'); log('TIER before', tierBefore); await pg.goto('http://127.0.0.1:8080/#/direct/!dm1:test/'); await pg.waitForTimeout(4000); const ed = pg.locator('[contenteditable="true"]'); log('EDITORS', await ed.count(), pg.url()); await ed.first().click(); await pg.keyboard.type('/converttoroom', { delay: 30 }); await pg.waitForTimeout(500); @@ -36,6 +38,12 @@ const HEADERS = ['GameCult', 'Projects', 'nibu', 'eureka', 'Nibu', 'Eureka', 'El await pg.waitForTimeout(30000); } for (const h of HEADERS) { const hl = pg.getByText(h, { exact: true }); if (await hl.count()) { await hl.first().click(); await pg.waitForTimeout(250); } } - log('TIER Slow DM', await tierOf('Slow DM'), '| Late DM', await tierOf('Late DM'), '| Friend DM', await tierOf('Friend DM')); - await b.close(); + const slow = await tierOf('Slow DM'), late = await tierOf('Late DM'), friend = await tierOf('Friend DM'); + log('TIER Slow DM', slow, '| Late DM', late, '| Friend DM', friend); + if (MODE === 'twodm') { expect('Slow DM is filed under Direct Messages', isDm(slow), slow); expect('Late DM is filed under Direct Messages', isDm(late), late); } + else if (MODE === 'convert') { + expect('Friend DM was filed under Direct Messages before the conversion', isDm(tierBefore), tierBefore); + expect('the converted room left Direct Messages and is still listed', !isDm(friend) && friend !== 'absent', friend); + } else expect('Slow DM is filed under Direct Messages after the reload', isDm(slow), slow); + await b.close(); process.exitCode = done(); })().catch((e) => { log('ERROR', String(e.stack || e)); process.exitCode = 1; }); diff --git a/harness/estate/run.sh b/harness/estate/run.sh index 379d28ca..65242ab0 100644 --- a/harness/estate/run.sh +++ b/harness/estate/run.sh @@ -8,19 +8,19 @@ npm run build > /tmp/build.log 2>&1 || { tail -30 /tmp/build.log; exit 1; } echo BUILD_OK $(git rev-parse HEAD) printf '{"defaultHomeserver":0,"homeserverList":["127.0.0.1:8080"],"allowCustomHomeservers":true,"featuredCommunities":{"spaces":["#gamecult:test"],"rooms":[],"servers":["test"]},"hashRouter":{"enabled":true,"basename":"/"}}' > dist/config.json mkdir -p /tmp/pw && cd /tmp/pw && npm init -y >/dev/null && npm i --no-audit --no-fund playwright@1.48.2 >/tmp/pw.log 2>&1 -cp /src/harness/estate/drive.cjs /src/harness/estate/drive-failures.cjs /src/harness/estate/drive-routes.cjs /tmp/pw/ +cp /src/harness/estate/check.cjs /src/harness/estate/drive.cjs /src/harness/estate/drive-failures.cjs /src/harness/estate/drive-routes.cjs /tmp/pw/ set +e for m in twodm reload convert; do HSLOG=/tmp/hslog-$m.txt NOROOT=0 node /src/harness/estate/fakehs.cjs /src/dist > /tmp/hs-$m.out 2>&1 & HS=$!; sleep 2 - MODE=$m node /tmp/pw/drive.cjs; echo "DRIVE $m RC=$?" + HSLOG=/tmp/hslog-$m.txt MODE=$m node /tmp/pw/drive.cjs; rc=$?; echo "DRIVE $m RC=$rc"; [ $rc -eq 0 ] || FAILED="$FAILED $m" kill $HS; wait $HS 2>/dev/null; sleep 1 echo "=== hs $m"; grep -E "MDIRECT_PUT|DELIVER|STATE_" /tmp/hs-$m.out; grep -E " JOIN " /tmp/hslog-$m.txt done failure() { m=$1; shift env "$@" HSLOG=/tmp/hslog-$m.txt NOROOT=0 node /src/harness/estate/fakehs.cjs /src/dist > /tmp/hs-$m.out 2>&1 & HS=$!; sleep 2 - MODE=$m node /tmp/pw/${DRIVER:-drive-failures}.cjs; echo "DRIVE $m RC=$?" + HSLOG=/tmp/hslog-$m.txt MODE=$m node /tmp/pw/${DRIVER:-drive-failures}.cjs; rc=$?; echo "DRIVE $m RC=$rc"; [ $rc -eq 0 ] || FAILED="$FAILED $m" kill $HS; wait $HS 2>/dev/null; sleep 1 echo "=== hs $m"; grep -E "MDIRECT|DELIVER|JOIN_|STATE_" /tmp/hs-$m.out; grep -E " JOIN " /tmp/hslog-$m.txt; } failure pjfail FAIL_MDIRECT=1 @@ -31,4 +31,5 @@ failure inboxadmin INVITER_ADMIN=1 failure inboxreload JOIN_DELAY=1 DRIVER=drive-routes failure deeplink DRIVER=drive-routes failure gw504 JOIN_504=1 -echo DONE +echo "FAILED_DRIVES:${FAILED:- none}" +[ -z "$FAILED" ]