Merge pull request 'Dings v3 behind client config switch dings.enabled (off by default)' (#69) from void/dings-v3 into dev

config.json

@@ -20,5 +20,9 @@
 
   "backgroundScene": {
     "tier": "off"
+  },
+
+  "dings": {
+    "enabled": false
   }
 }

public/dings/README.md

@@ -0,0 +1,17 @@
+# Dings
+
+Pre-rendered earcons, served at `public/dings/` and read at runtime by `src/app/features/dings`.
+Aqua renders the clips; nothing here is synthesised in the browser.
+
+    dings/manifest.json
+    dings/senders/<mxid-localpart>/<motif>.ogg
+    dings/neutral/<motif>.ogg
+
+`manifest.json` is `{ "neutral": { <motif>: "<path>" }, "senders": { <localpart>: { <motif>: "<path>" } } }`
+with motif one of `mention`, `dm`, `message`, `invite`; paths are relative to this folder, `.ogg` or `.mp3`.
+A sender's own clip wins, else the neutral clip, else silence. Without a `manifest.json` the client keeps
+its stock notification sounds; once one exists, a motif with no clip is silent.
+
+The set here is Aqua's v3 (AquaSynth PR #22, staged at minds.gamecult.org/aqua/dings-voices-70a9795),
+Nibu on proposal 1 (static-filter square). Nobody has heard it. The client plays none of it unless the
+client config sets `"dings": { "enabled": true }` (default off, as in `config.json`).

public/dings/manifest.json

@@ -0,0 +1,58 @@
+{
+  "neutral": {
+    "mention": "neutral/mention.ogg",
+    "dm": "neutral/dm.ogg",
+    "message": "neutral/message.ogg",
+    "invite": "neutral/invite.ogg"
+  },
+  "senders": {
+    "aqua": {
+      "mention": "senders/aqua/mention.ogg",
+      "dm": "senders/aqua/dm.ogg",
+      "message": "senders/aqua/message.ogg",
+      "invite": "senders/aqua/invite.ogg"
+    },
+    "eureka": {
+      "mention": "senders/eureka/mention.ogg",
+      "dm": "senders/eureka/dm.ogg",
+      "message": "senders/eureka/message.ogg",
+      "invite": "senders/eureka/invite.ogg"
+    },
+    "heimdall": {
+      "mention": "senders/heimdall/mention.ogg",
+      "dm": "senders/heimdall/dm.ogg",
+      "message": "senders/heimdall/message.ogg",
+      "invite": "senders/heimdall/invite.ogg"
+    },
+    "kiko": {
+      "mention": "senders/kiko/mention.ogg",
+      "dm": "senders/kiko/dm.ogg",
+      "message": "senders/kiko/message.ogg",
+      "invite": "senders/kiko/invite.ogg"
+    },
+    "libby": {
+      "mention": "senders/libby/mention.ogg",
+      "dm": "senders/libby/dm.ogg",
+      "message": "senders/libby/message.ogg",
+      "invite": "senders/libby/invite.ogg"
+    },
+    "nibu": {
+      "mention": "senders/nibu/mention.ogg",
+      "dm": "senders/nibu/dm.ogg",
+      "message": "senders/nibu/message.ogg",
+      "invite": "senders/nibu/invite.ogg"
+    },
+    "void": {
+      "mention": "senders/void/mention.ogg",
+      "dm": "senders/void/dm.ogg",
+      "message": "senders/void/message.ogg",
+      "invite": "senders/void/invite.ogg"
+    },
+    "weksa": {
+      "mention": "senders/weksa/mention.ogg",
+      "dm": "senders/weksa/dm.ogg",
+      "message": "senders/weksa/message.ogg",
+      "invite": "senders/weksa/invite.ogg"
+    }
+  }
+}

src/app/features/dings/dings.test.ts

@@ -0,0 +1,144 @@
+import { describe, expect, it, vi } from 'vitest';
+import { DING_VOLUME, createDingPlayer, motifFor, parseManifest, pickDing } from './dings';
+
+const manifest = parseManifest({
+  neutral: { mention: 'neutral/mention.ogg', message: 'neutral/message.ogg' },
+  senders: {
+    eureka: { mention: 'eureka/mention.ogg', message: 'eureka/message.ogg', dm: 'eureka/dm.ogg' },
+    nibu: { message: 'nibu/message.ogg' },
+  },
+})!;
+
+describe('dings manifest', () => {
+  it('is read defensively: junk is no manifest, bad paths are dropped', () => {
+    expect(parseManifest(undefined)).toBeUndefined();
+    expect(parseManifest('x')).toBeUndefined();
+    expect(parseManifest({ neutral: 3, senders: [] })).toBeUndefined();
+    const m = parseManifest({
+      neutral: { message: '../../etc/passwd.ogg', mention: 'https://evil.example/a.ogg' },
+      senders: { void: { message: 'void/message.wav', dm: 'void/dm.ogg', mention: '/abs.ogg' } },
+    })!;
+    expect(m.neutral).toEqual({});
+    expect(m.senders.void).toEqual({ dm: 'void/dm.ogg' });
+  });
+});
+
+describe('manifest guards', () => {
+  it('rejects a dot segment in the middle of a path', () => {
+    const m = parseManifest({
+      neutral: { message: 'a/../b.ogg', mention: 'a/b.ogg' },
+      senders: {},
+    })!;
+    expect(m.neutral).toEqual({ mention: 'a/b.ogg' });
+  });
+
+  it('is not fooled by an array of senders', () => {
+    expect(parseManifest({ neutral: { message: 'a.ogg' }, senders: [] })).toBeUndefined();
+  });
+
+  it('ignores motifs a clip set only inherits', () => {
+    const inherited = Object.create({ message: 'x/message.ogg' });
+    const m = parseManifest({ neutral: inherited, senders: {} })!;
+    expect(m.neutral).toEqual({});
+  });
+
+  it('gives object-prototype names no clip of their own', () => {
+    const m = parseManifest({ neutral: { message: 'n/message.ogg' }, senders: {} })!;
+    ['__proto__', 'constructor', 'toString', 'hasOwnProperty'].forEach((who) => {
+      expect(pickDing(m, who, 'message')?.path).toBe('n/message.ogg');
+    });
+  });
+
+  it('reports the motif volume in the pick itself', () => {
+    expect(pickDing(manifest, 'eureka', 'message')?.volume).toBe(DING_VOLUME.message);
+    expect(pickDing(manifest, 'stranger', 'mention')?.volume).toBe(1);
+  });
+});
+
+describe('which ding plays', () => {
+  it('uses the sender instrument for the motif', () => {
+    expect(pickDing(manifest, 'eureka', 'mention')).toEqual({
+      path: 'eureka/mention.ogg',
+      volume: DING_VOLUME.mention,
+    });
+  });
+
+  it('falls back to the neutral ding for an unknown sender', () => {
+    expect(pickDing(manifest, 'stranger', 'message')?.path).toBe('neutral/message.ogg');
+  });
+
+  it('falls back to neutral when the sender has no clip for that motif', () => {
+    expect(pickDing(manifest, 'nibu', 'mention')?.path).toBe('neutral/mention.ogg');
+  });
+
+  it('is silent when neither the sender nor neutral has a clip', () => {
+    expect(pickDing(manifest, 'eureka', 'invite')).toBeUndefined();
+    expect(pickDing(manifest, 'stranger', 'dm')).toBeUndefined();
+  });
+
+  it('is loud for mentions and DMs and quiet for ordinary messages', () => {
+    expect(DING_VOLUME.mention).toBe(1);
+    expect(DING_VOLUME.dm).toBe(1);
+    expect(DING_VOLUME.message).toBeLessThan(DING_VOLUME.invite);
+    expect(DING_VOLUME.invite).toBeLessThan(DING_VOLUME.mention);
+  });
+
+  it('names the motif: invite over mention over DM over message', () => {
+    expect(motifFor({ invite: true, highlight: true, direct: true })).toBe('invite');
+    expect(motifFor({ highlight: true, direct: true })).toBe('mention');
+    expect(motifFor({ highlight: false, direct: true })).toBe('dm');
+    expect(motifFor({ highlight: false, direct: false })).toBe('message');
+  });
+});
+
+describe('audio unlock', () => {
+  const make = () => {
+    const listeners = new Map<string, () => void>();
+    const target = {
+      addEventListener: (t: string, f: () => void) => listeners.set(t, f),
+      removeEventListener: (t: string) => listeners.delete(t),
+    };
+    const played: Array<{ url: string; volume: number }> = [];
+    const player = createDingPlayer((url) => {
+      const clip = { volume: 1, play: vi.fn(async () => undefined) };
+      played.push({
+        url,
+        get volume() {
+          return clip.volume;
+        },
+      } as never);
+      return clip;
+    }, target);
+    return { listeners, player, played };
+  };
+
+  it('does not play before the first user gesture, and plays after it', () => {
+    const { listeners, player, played } = make();
+    expect(player.play('a.ogg', 1)).toBe(false);
+    expect(played).toHaveLength(0);
+    listeners.get('pointerdown')!();
+    expect(player.play('a.ogg', 0.3)).toBe(true);
+    expect(played).toHaveLength(1);
+    expect(played[0].volume).toBe(0.3);
+  });
+
+  it('stops listening once unlocked', () => {
+    const { listeners, player } = make();
+    listeners.get('keydown')!();
+    expect(listeners.size).toBe(0);
+    expect(player.unlocked()).toBe(true);
+  });
+
+  it('a clip the browser refuses does not throw', async () => {
+    const target = {
+      addEventListener: (_t: string, f: () => void) => f(),
+      removeEventListener: () => undefined,
+    };
+    const player = createDingPlayer(
+      () => ({ volume: 1, play: () => Promise.reject(new Error('NotAllowedError')) }),
+      target
+    );
+    expect(() => player.play('a.ogg', 1)).not.toThrow();
+    await Promise.resolve();
+  });
+});

src/app/features/dings/dings.ts

@@ -0,0 +1,100 @@
+// Dings: pre-rendered earcons shipped as static assets, `dings/<mxid-localpart>/<motif>.ogg` plus
+// `dings/manifest.json` (docs/gamecult/ratatoskr-target.md, Agreements). The sender picks the
+// instrument, the event picks the motif. This file is the whole decision; the client only asks it.
+
+export type Motif = 'mention' | 'dm' | 'message' | 'invite';
+type Clips = Partial<Record<Motif, string>>;
+export type DingManifest = { neutral: Clips; senders: Record<string, Clips> };
+
+/** Client config `dings`. Off unless `enabled` is exactly true; nothing about dings loads otherwise. */
+export type DingsConfig = { enabled?: boolean };
+export const dingsEnabled = (config: DingsConfig | undefined): boolean => config?.enabled === true;
+
+const MOTIFS: Motif[] = ['mention', 'dm', 'message', 'invite'];
+
+// Mentions and DMs loud, an ordinary message quiet.
+export const DING_VOLUME: Record<Motif, number> = {
+  mention: 1,
+  dm: 1,
+  invite: 0.7,
+  message: 0.3,
+};
+
+const isRecord = (v: unknown): v is Record<string, unknown> =>
+  typeof v === 'object' && v !== null && !Array.isArray(v);
+
+// A clip path stays inside dings/: relative, ogg or mp3 (not wav), no dot segments, no scheme.
+const isClipPath = (v: unknown): v is string =>
+  typeof v === 'string' &&
+  /^[A-Za-z0-9_-]+(\/[A-Za-z0-9_.-]+)*\.(ogg|mp3)$/.test(v) &&
+  !v.split('/').includes('..');
+
+const readClips = (v: unknown): Clips => {
+  const clips: Clips = {};
+  if (!isRecord(v)) return clips;
+  MOTIFS.forEach((m) => {
+    if (Object.prototype.hasOwnProperty.call(v, m) && isClipPath(v[m])) clips[m] = v[m] as string;
+  });
+  return clips;
+};
+
+/** The manifest is JSON from outside the typed world: read it defensively, undefined if unusable. */
+export const parseManifest = (json: unknown): DingManifest | undefined => {
+  if (!isRecord(json) || !isRecord(json.neutral) || !isRecord(json.senders)) return undefined;
+  const senders: Record<string, Clips> = Object.create(null);
+  Object.keys(json.senders).forEach((localpart) => {
+    senders[localpart] = readClips(
+      json.senders && (json.senders as Record<string, unknown>)[localpart]
+    );
+  });
+  return { neutral: readClips(json.neutral), senders };
+};
+
+/** The sender's own clip, else the neutral one, else nothing: silence. */
+export const pickDing = (
+  manifest: DingManifest,
+  localpart: string,
+  motif: Motif
+): { path: string; volume: number } | undefined => {
+  const own = Object.prototype.hasOwnProperty.call(manifest.senders, localpart)
+    ? manifest.senders[localpart][motif]
+    : undefined;
+  const path = own ?? manifest.neutral[motif];
+  return path === undefined ? undefined : { path, volume: DING_VOLUME[motif] };
+};
+
+export const motifFor = (e: { invite?: boolean; highlight: boolean; direct: boolean }): Motif => {
+  if (e.invite) return 'invite';
+  if (e.highlight) return 'mention';
+  return e.direct ? 'dm' : 'message';
+};
+
+type Clip = { volume: number; play: () => Promise<void> };
+type GestureTarget = {
+  addEventListener: (type: string, listener: () => void) => void;
+  removeEventListener: (type: string, listener: () => void) => void;
+};
+const GESTURES = ['pointerdown', 'keydown', 'touchend'];
+
+/**
+ * Browsers refuse audio until the first user gesture. The player stays silent until one has
+ * happened, then plays; a clip the browser still refuses is dropped, never thrown.
+ */
+export const createDingPlayer = (make: (url: string) => Clip, target: GestureTarget) => {
+  let unlocked = false;
+  const unlock = () => {
+    unlocked = true;
+    GESTURES.forEach((g) => target.removeEventListener(g, unlock));
+  };
+  GESTURES.forEach((g) => target.addEventListener(g, unlock));
+  return {
+    unlocked: () => unlocked,
+    play(url: string, volume: number): boolean {
+      if (!unlocked) return false;
+      const clip = make(url);
+      clip.volume = volume;
+      clip.play().catch(() => undefined);
+      return true;
+    },
+  };
+};

src/app/features/dings/manifest.test.ts

@@ -0,0 +1,94 @@
+import { readFileSync, existsSync } from 'node:fs';
+import { createHash } from 'node:crypto';
+import { join } from 'node:path';
+import { describe, expect, it } from 'vitest';
+import { DingManifest, Motif, dingsEnabled, parseManifest, pickDing } from './dings';
+
+// The staged v3 set (Aqua, AquaSynth PR #22). Nobody has heard it; these tests check the manifest
+// against this code, not the sound.
+const DINGS = join(__dirname, '../../../../public/dings');
+const raw = JSON.parse(readFileSync(join(DINGS, 'manifest.json'), 'utf8'));
+const manifest = parseManifest(raw) as DingManifest;
+const MOTIFS: Motif[] = ['mention', 'dm', 'message', 'invite'];
+const VOICES = ['aqua', 'eureka', 'heimdall', 'kiko', 'libby', 'nibu', 'void', 'weksa'];
+
+// Vorbis-in-Ogg duration: last page's granule position over the sample rate in the id header.
+const oggMs = (file: string): number => {
+  const b = readFileSync(join(DINGS, file));
+  const id = b.indexOf('\x01vorbis', 0, 'latin1');
+  const rate = b.readUInt32LE(id + 12);
+  const last = b.lastIndexOf('OggS', b.length, 'latin1');
+  return (Number(b.readBigUInt64LE(last + 6)) / rate) * 1000;
+};
+
+describe('dings switch', () => {
+  it('is off unless the config says exactly true', () => {
+    expect(dingsEnabled(undefined)).toBe(false);
+    expect(dingsEnabled({})).toBe(false);
+    expect(dingsEnabled({ enabled: false })).toBe(false);
+    expect(dingsEnabled({ enabled: 'true' as unknown as boolean })).toBe(false);
+    expect(dingsEnabled({ enabled: true })).toBe(true);
+  });
+
+  it('ships off in config.json', () => {
+    const config = JSON.parse(readFileSync(join(__dirname, '../../../../config.json'), 'utf8'));
+    expect(config.dings).toEqual({ enabled: false });
+    expect(dingsEnabled(config.dings)).toBe(false);
+  });
+});
+
+describe('v3 manifest against the parser', () => {
+  it('survives parseManifest whole: no path or motif is dropped', () => {
+    expect(manifest.neutral).toEqual(raw.neutral);
+    expect(Object.keys(manifest.senders).sort()).toEqual(VOICES);
+    VOICES.forEach((v) => expect(manifest.senders[v]).toEqual(raw.senders[v]));
+    MOTIFS.forEach((m) => expect(manifest.neutral[m]).toBeDefined());
+  });
+
+  it('names a file that exists for every clip, and no file the manifest omits', () => {
+    const paths = [
+      ...Object.values(manifest.neutral),
+      ...Object.values(manifest.senders).flatMap((c) => Object.values(c)),
+    ];
+    expect(paths).toHaveLength(4 + VOICES.length * 4);
+    paths.forEach((p) => expect(existsSync(join(DINGS, p as string)), p).toBe(true));
+  });
+
+  it('maps a sender localpart to that sender folder, per motif', () => {
+    VOICES.forEach((v) =>
+      MOTIFS.forEach((m) => expect(pickDing(manifest, v, m)?.path).toBe(`senders/${v}/${m}.ogg`))
+    );
+  });
+
+  it('an unknown sender and an invite get the neutral clip', () => {
+    MOTIFS.forEach((m) => expect(pickDing(manifest, 'someone', m)?.path).toBe(`neutral/${m}.ogg`));
+  });
+
+  it('Nibu carries pick 3 (8 kHz 5-bit smoothed sine) for every event: senders/nibu, never the proposals folder', () => {
+    MOTIFS.forEach((m) => {
+      const path = pickDing(manifest, 'nibu', m)?.path as string;
+      expect(path).toBe(`senders/nibu/${m}.ogg`);
+      expect(path).not.toContain('proposals');
+    });
+  });
+
+  it("Nibu's four files are AquaSynth's proposals/nibu3 (sha256 prefix)", () => {
+    const want: Record<Motif, string> = {
+      mention: '3833f504903f',
+      dm: 'ad4839400f2c',
+      message: '6ec6fc893b94',
+      invite: 'aefb6ff79c42',
+    };
+    MOTIFS.forEach((m) => {
+      const hash = createHash('sha256').update(readFileSync(join(DINGS, `senders/nibu/${m}.ogg`)));
+      expect(hash.digest('hex').slice(0, 12)).toBe(want[m]);
+    });
+  });
+
+  it('keeps the clip lengths the design assumes: message under 400 ms, nothing over 600 ms', () => {
+    VOICES.forEach((v) => {
+      expect(oggMs(`senders/${v}/message.ogg`)).toBeLessThan(400);
+      MOTIFS.forEach((m) => expect(oggMs(`senders/${v}/${m}.ogg`)).toBeLessThan(601));
+    });
+  });
+});

src/app/features/dings/useDings.ts

@@ -0,0 +1,55 @@
+import { useCallback, useEffect, useMemo, useRef } from 'react';
+import { useClientConfig } from '../../hooks/useClientConfig';
+import { trimTrailingSlash } from '../../utils/common';
+import { getMxIdLocalPart } from '../../utils/matrix';
+import {
+  DingManifest,
+  Motif,
+  createDingPlayer,
+  dingsEnabled,
+  parseManifest,
+  pickDing,
+} from './dings';
+
+const DINGS_ROOT = `${trimTrailingSlash(import.meta.env.BASE_URL)}/public/dings`;
+
+/**
+ * Plays a ding for a sender and motif. The manifest is read once at runtime. Returns false when
+ * there is no manifest (nothing rendered yet), so the caller keeps the stock sound; once there is
+ * one, a sender or motif with no clip is silence, as the design says, and the return is true.
+ * The client config's `dings.enabled` is the switch (default off): off, nothing is fetched, no
+ * gesture listener is added, and the return is always false, so the stock sound plays.
+ */
+export const useDings = () => {
+  const enabled = dingsEnabled(useClientConfig().dings);
+  const manifestRef = useRef<DingManifest>();
+  const player = useMemo(
+    () => (enabled ? createDingPlayer((url) => new Audio(url), window) : undefined),
+    [enabled]
+  );
+
+  useEffect(() => {
+    if (!enabled) return undefined;
+    let live = true;
+    fetch(`${DINGS_ROOT}/manifest.json`)
+      .then((r) => (r.ok ? r.json() : undefined))
+      .then((json) => {
+        if (live) manifestRef.current = parseManifest(json);
+      })
+      .catch(() => undefined);
+    return () => {
+      live = false;
+    };
+  }, [enabled]);
+
+  return useCallback(
+    (sender: string, motif: Motif): boolean => {
+      const manifest = manifestRef.current;
+      if (!player || !manifest) return false;
+      const ding = pickDing(manifest, getMxIdLocalPart(sender) ?? sender, motif);
+      if (ding) player.play(`${DINGS_ROOT}/${ding.path}`, ding.volume);
+      return true;
+    },
+    [player]
+  );
+};

src/app/hooks/useClientConfig.ts

@@ -1,4 +1,5 @@
 import { createContext, useContext } from 'react';
+import type { DingsConfig } from '../features/dings/dings';
 import type { BackgroundSceneConfig } from '../features/scene/tier';
 
 export type HashRouterConfig = {
@@ -22,6 +23,9 @@ export type ClientConfig = {
 
   backgroundScene?: BackgroundSceneConfig;
 
+  // Per-Mind notification earcons (src/app/features/dings). Off unless `enabled: true`.
+  dings?: DingsConfig;
+
   // Dev preview only: the build answers a pretend homeserver in-page (src/preview-demo).
   previewDemo?: boolean;
 };

src/app/pages/client/ClientNonUIFeatures.tsx

@@ -26,6 +26,9 @@ import { getMxIdLocalPart, mxcUrlToHttp } from '../../utils/matrix';
 import { useSelectedRoom } from '../../hooks/router/useSelectedRoom';
 import { useInboxNotificationsSelected } from '../../hooks/router/useInbox';
 import { useMediaAuthentication } from '../../hooks/useMediaAuthentication';
+import { mDirectAtom } from '../../state/mDirectList';
+import { motifFor } from '../../features/dings/dings';
+import { useDings } from '../../features/dings/useDings';
 
 function SystemEmojiFeature() {
   const [twitterEmoji] = useSetting(settingsAtom, 'twitterEmoji');
@@ -85,6 +88,7 @@ function InviteNotifications() {
   const navigate = useNavigate();
   const [showNotifications] = useSetting(settingsAtom, 'showNotifications');
   const [notificationSound] = useSetting(settingsAtom, 'isNotificationSounds');
+  const playDing = useDings();
 
   const notify = useCallback(
     (count: number) => {
@@ -115,10 +119,20 @@ function InviteNotifications() {
       }
 
       if (notificationSound) {
-        playSound();
+        // The sender of an invite is not in the invite list, so it gets the neutral invite ding.
+        if (!playDing('', 'invite')) playSound();
       }
     }
-  }, [mx, invites, perviousInviteLen, showNotifications, notificationSound, notify, playSound]);
+  }, [
+    mx,
+    invites,
+    perviousInviteLen,
+    showNotifications,
+    notificationSound,
+    notify,
+    playSound,
+    playDing,
+  ]);
 
   return (
     // eslint-disable-next-line jsx-a11y/media-has-caption
@@ -136,6 +150,8 @@ function MessageNotifications() {
   const useAuthentication = useMediaAuthentication();
   const [showNotifications] = useSetting(settingsAtom, 'showNotifications');
   const [notificationSound] = useSetting(settingsAtom, 'isNotificationSounds');
+  const mDirects = useAtomValue(mDirectAtom);
+  const playDing = useDings();
 
   const navigate = useNavigate();
   const notificationSelected = useInboxNotificationsSelected();
@@ -227,7 +243,11 @@ function MessageNotifications() {
       }
 
       if (notificationSound) {
-        playSound();
+        const motif = motifFor({
+          highlight: unreadInfo.highlight > 0,
+          direct: mDirects.has(room.roomId),
+        });
+        if (!playDing(sender, motif)) playSound();
       }
     };
     mx.on(RoomEvent.Timeline, handleTimelineEvent);
@@ -240,6 +260,8 @@ function MessageNotifications() {
     notificationSelected,
     showNotifications,
     playSound,
+    playDing,
+    mDirects,
     notify,
     selectedRoomId,
     useAuthentication,

vite.config.js

@@ -46,6 +46,10 @@ const copyFiles = {
       src: 'public/poses',
       dest: 'public/',
     },
+    {
+      src: 'public/dings',
+      dest: 'public/',
+    },
   ],
 };