diff --git a/call.config.json b/call.config.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/call.config.json @@ -0,0 +1 @@ +{} diff --git a/package-lock.json b/package-lock.json index 4a47822c..6095a684 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,6 @@ "@atlaskit/pragmatic-drag-and-drop-auto-scroll": "1.3.0", "@atlaskit/pragmatic-drag-and-drop-hitbox": "1.0.3", "@fontsource/inter": "4.5.14", - "@matrix-org/matrix-sdk-crypto-wasm": "18.3.1", "@tanstack/react-query": "5.24.1", "@tanstack/react-query-devtools": "5.24.1", "@tanstack/react-virtual": "3.2.0", @@ -33,7 +32,7 @@ "emojibase-data": "15.3.2", "file-saver": "2.0.5", "focus-trap-react": "10.0.2", - "folds": "2.7.1", + "folds": "2.7.3", "html-dom-parser": "4.0.0", "html-react-parser": "4.2.0", "i18next": "23.12.2", @@ -48,6 +47,7 @@ "matrix-js-sdk": "41.7.0", "matrix-widget-api": "1.20.0", "millify": "6.1.0", + "neurone": "0.0.9", "pdfjs-dist": "4.2.67", "prismjs": "1.30.0", "react": "18.2.0", @@ -68,7 +68,7 @@ "ua-parser-js": "1.0.35" }, "devDependencies": { - "@element-hq/element-call-embedded": "0.26.1", + "@element-hq/element-call-embedded": "0.28.0", "@esbuild-plugins/node-globals-polyfill": "0.2.3", "@rollup/plugin-inject": "5.0.3", "@rollup/plugin-wasm": "6.1.1", @@ -1776,9 +1776,9 @@ } }, "node_modules/@element-hq/element-call-embedded": { - "version": "0.26.1", - "resolved": "https://registry.npmjs.org/@element-hq/element-call-embedded/-/element-call-embedded-0.26.1.tgz", - "integrity": "sha512-0zAC8Qh+xNLTqiC7yiosu2NQ7Fw36xR7Gimgmp2WZjTbiPf8b7DmYOBZk0sWFzYbrHHZTeB2wWPDmS3TgYq51g==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@element-hq/element-call-embedded/-/element-call-embedded-0.28.0.tgz", + "integrity": "sha512-wM5tLXrSCLbveGlvQg1Pj6dNAp75mhJe2Zr3Vb8dtK6z9tuuFMcsn1q1Gjy1KYWQdzBzKK2wLc1O8H7DzYZRHQ==", "dev": true }, "node_modules/@emotion/hash": { @@ -7908,9 +7908,9 @@ } }, "node_modules/folds": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/folds/-/folds-2.7.1.tgz", - "integrity": "sha512-A1XGXJ4hp1W0cpFUIjYj9gPSzKVxqj9BlTBJTqmmtTUiEFDzXvhBITJdcqmlDsmcd756tu/iygSAzdky5fDcrg==", + "version": "2.7.3", + "resolved": "https://registry.npmjs.org/folds/-/folds-2.7.3.tgz", + "integrity": "sha512-oMdMOBmU5RCUI4nmQKiEKpJNh+ZmCijvR4+aalk/btZPp1jIRGha7MXBb2NFOy1y9od+RynNFp+bw6E7GMATBA==", "license": "Apache-2.0", "peerDependencies": { "@vanilla-extract/css": "1.9.2", @@ -10390,6 +10390,24 @@ "integrity": "sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g==", "dev": true }, + "node_modules/neurone": { + "version": "0.0.9", + "resolved": "https://registry.npmjs.org/neurone/-/neurone-0.0.9.tgz", + "integrity": "sha512-A//2eeyxJLwuDGCZh7N6jNy0tloQ1pgY/0pNna5SrOsj5UPKb2W7U3wE9+2q7sN2kT+0SZr2xijfi06EGkrlyQ==", + "license": "Apache-2.0", + "dependencies": { + "@matrix-org/matrix-sdk-crypto-wasm": "18.3.1", + "another-json": "0.2.0", + "bs58": "6.0.0", + "idb": "8.0.0" + } + }, + "node_modules/neurone/node_modules/idb": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/idb/-/idb-8.0.0.tgz", + "integrity": "sha512-l//qvlAKGmQO31Qn7xdzagVPPaHTxXx199MhrAFuVBTPqydcPYBWjkrbv4Y0ktB+GmWOiwHl237UUOrLmQxLvw==", + "license": "ISC" + }, "node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", diff --git a/package.json b/package.json index 8530bf26..5915bfb9 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,6 @@ "@atlaskit/pragmatic-drag-and-drop-auto-scroll": "1.3.0", "@atlaskit/pragmatic-drag-and-drop-hitbox": "1.0.3", "@fontsource/inter": "4.5.14", - "@matrix-org/matrix-sdk-crypto-wasm": "18.3.1", "@tanstack/react-query": "5.24.1", "@tanstack/react-query-devtools": "5.24.1", "@tanstack/react-virtual": "3.2.0", @@ -57,7 +56,7 @@ "emojibase-data": "15.3.2", "file-saver": "2.0.5", "focus-trap-react": "10.0.2", - "folds": "2.7.1", + "folds": "2.7.3", "html-dom-parser": "4.0.0", "html-react-parser": "4.2.0", "i18next": "23.12.2", @@ -72,6 +71,7 @@ "matrix-js-sdk": "41.7.0", "matrix-widget-api": "1.20.0", "millify": "6.1.0", + "neurone": "0.0.9", "pdfjs-dist": "4.2.67", "prismjs": "1.30.0", "react": "18.2.0", @@ -92,7 +92,7 @@ "ua-parser-js": "1.0.35" }, "devDependencies": { - "@element-hq/element-call-embedded": "0.26.1", + "@element-hq/element-call-embedded": "0.28.0", "@esbuild-plugins/node-globals-polyfill": "0.2.3", "@rollup/plugin-inject": "5.0.3", "@rollup/plugin-wasm": "6.1.1", diff --git a/src/app/components/CallEmbedProvider.tsx b/src/app/components/CallEmbedProvider.tsx index a6f6dec5..d22b20da 100644 --- a/src/app/components/CallEmbedProvider.tsx +++ b/src/app/components/CallEmbedProvider.tsx @@ -17,15 +17,7 @@ import { Text, toRem, } from 'folds'; -import { - EventTimelineSetHandlerMap, - EventType, - RelationType, - Room, - RoomEvent, -} from 'matrix-js-sdk'; -import { IRTCNotificationContent, RTCNotificationType } from 'matrix-js-sdk/lib/matrixrtc/types'; -import { CryptoBackend } from 'matrix-js-sdk/lib/common-crypto/CryptoBackend'; +import { putSend } from 'neurone/lib/api/matrix/client/v3/rooms'; import { CallEmbedContextProvider, CallEmbedRefContextProvider, @@ -39,57 +31,66 @@ import { callChatAtom, callEmbedAtom } from '../state/callEmbed'; import { CallEmbed } from '../plugins/call'; import { useSelectedRoom } from '../hooks/router/useSelectedRoom'; import { ScreenSize, useScreenSizeContext } from '../hooks/useScreenSize'; -import { useMatrixClient } from '../hooks/useMatrixClient'; import CallSound from '../../../public/sound/call.ogg'; import { useCallMembersChange, useCallSession } from '../hooks/useCall'; -import { useRoomAvatar, useRoomName } from '../hooks/useRoomMeta'; import { mDirectAtom } from '../state/mDirectList'; import { useMediaAuthentication } from '../hooks/useMediaAuthentication'; -import { mxcUrlToHttp } from '../utils/matrix'; import { RoomAvatar, RoomIcon } from './room-avatar'; import { useRoomNavigate } from '../hooks/useRoomNavigate'; -import { getStateEvent } from '../utils/room'; -import { StateEvent } from '../../types/matrix/room'; -import { getPowersLevelFromMatrixEvent } from '../hooks/usePowerLevels'; -import { getRoomCreatorsForRoomId } from '../hooks/useRoomCreators'; -import { getRoomPermissionsAPI } from '../hooks/useRoomPermissions'; +import { CallEventType, MessageEvent } from '../../types/matrix/room'; import { useLivekitSupport } from '../hooks/useLivekitSupport'; import { CallAvatarAnimation } from '../styles/Animations.css'; import { webRTCSupported } from '../utils/rtc'; +import { useClient } from '../hooks/useClient'; +import { useGetRoomName } from '../hooks/storage/useGetRoomName'; +import { useRoomHeroes } from '../hooks/storage/useRoomHeros'; +import { useGetRoomAvatar } from '../hooks/storage/useGetRoomAvatar'; +import { mxcToHttp } from '../../client/mxcToHttp'; +import { RelationType } from '../../types/matrix/common'; +import { ensureEncryptionSession } from '../hooks/useEnsureRoomEncryptionSession'; type IncomingCallInfo = { - room: Room; + roomId: string; sender: string; senderTs: number; lifetime: number; intent?: string; - notificationType: RTCNotificationType; + silent?: boolean; refEventId: string; }; type IncomingCallProps = { dm: boolean; info: IncomingCallInfo; onIgnore: () => void; - onAnswer: (room: Room, video: boolean) => void; - onReject: (room: Room, eventId: string) => void; + onAnswer: (roomId: string, video: boolean) => void; + onReject: (roomId: string, eventId: string) => void; }; function IncomingCall({ dm, info, onIgnore, onAnswer, onReject }: IncomingCallProps) { - const mx = useMatrixClient(); + const ct = useClient(); const useAuthentication = useMediaAuthentication(); const livekitSupported = useLivekitSupport(); const rtcSupported = webRTCSupported(); const canAnswer = livekitSupported && rtcSupported; - const { room } = info; + const { roomId } = info; const audioRef = useRef(null); - const roomName = useRoomName(room); - const roomAvatar = useRoomAvatar(room, dm); + const getRoomHeroes = useRoomHeroes(); + const getRoomName = useGetRoomName(getRoomHeroes); + const getRoomAvatar = useGetRoomAvatar(getRoomHeroes); + + const roomName = getRoomName(roomId); + const roomAvatar = getRoomAvatar(roomId); + const avatarUrl = roomAvatar - ? mxcUrlToHttp(mx, roomAvatar, useAuthentication, 96, 96, 'crop') ?? undefined + ? mxcToHttp(ct.session.baseUrl, roomAvatar, useAuthentication).thumbnail({ + width: 96, + height: 96, + method: 'crop', + }) : undefined; - const session = useCallSession(room); + const session = useCallSession(roomId); useCallMembersChange( session, useCallback( @@ -108,10 +109,10 @@ function IncomingCall({ dm, info, onIgnore, onAnswer, onReject }: IncomingCallPr }, []); useEffect(() => { - if (info.notificationType === 'ring') { + if (!info.silent) { playSound(); } - }, [playSound, info.notificationType]); + }, [playSound, info.silent]); return ( <> @@ -134,14 +135,14 @@ function IncomingCall({ dm, info, onIgnore, onAnswer, onReject }: IncomingCallPr ( )} @@ -181,7 +182,7 @@ function IncomingCall({ dm, info, onIgnore, onAnswer, onReject }: IncomingCallPr variant="Success" size="400" radii="400" - onClick={() => onAnswer(room, info.intent === 'video')} + onClick={() => onAnswer(roomId, info.intent === 'video')} before={ (dm ? onReject(room, info.refEventId) : onIgnore())} + onClick={() => (dm ? onReject(roomId, info.refEventId) : onIgnore())} before={} > @@ -226,117 +227,140 @@ type IncomingCallListenerProps = { joined?: boolean; }; function IncomingCallListener({ callEmbed, joined }: IncomingCallListenerProps) { - const mx = useMatrixClient(); + const ct = useClient(); const directs = useAtomValue(mDirectAtom); const { navigateRoom } = useRoomNavigate(); const [callInfo, setCallInfo] = useState(); - const dm = callInfo ? directs.has(callInfo.room.roomId) : false; - const startCall = useCallStart(dm); - - const handleTimelineEvent: EventTimelineSetHandlerMap[RoomEvent.Timeline] = useCallback( - async (event, room, toStartOfTimeline, removed, data) => { - // only process rtc notification reference events. - // we do not want to wait to decrypt all events. - if (event.getRelation()?.rel_type !== RelationType.Reference) return; - if (room?.isCallRoom()) return; - - if (event.isEncrypted()) { - if (!event.isBeingDecrypted()) { - await event.attemptDecryption(mx.getCrypto() as CryptoBackend); - } - await event.getDecryptionPromise(); - } - - if ( - !room || - event.getType() !== EventType.RTCNotification || - event.getSender() === mx.getSafeUserId() || - !data.liveEvent - ) { - return; - } - - const sender = event.getSender(); - const content = event.getContent(); - const senderTs = - content.sender_ts - event.getTs() > 20000 ? event.getTs() : content.sender_ts; - const lifetime = Math.min(content.lifetime, 120000); - const notificationType = content.notification_type; - const relation = - event.getRelation()?.rel_type === RelationType.Reference ? event.getRelation() : undefined; - const refEventId = relation?.event_id; - - const mention = - content['m.mentions']?.room || - content['m.mentions']?.user_ids?.includes(mx.getSafeUserId()); - if (!sender || !refEventId || !mention || Date.now() >= senderTs + lifetime) { - return; - } - - const powerLevelsEvent = getStateEvent(room, StateEvent.RoomPowerLevels); - const powerLevels = getPowersLevelFromMatrixEvent(powerLevelsEvent); - const creators = getRoomCreatorsForRoomId(mx, room.roomId); - const permissions = getRoomPermissionsAPI(creators, powerLevels); - - const hasCallPermission = permissions.stateEvent( - StateEvent.GroupCallMemberPrefix, - mx.getSafeUserId() - ); - if (!hasCallPermission) return; - - const info: IncomingCallInfo = { - room, - sender, - senderTs, - lifetime, - intent: - 'm.call.intent' in content && typeof content['m.call.intent'] === 'string' - ? content['m.call.intent'] - : undefined, - notificationType, - refEventId, - }; - - setCallInfo(info); - }, - [mx] - ); - - useEffect(() => { - mx.on(RoomEvent.Timeline, handleTimelineEvent); - return () => { - mx.removeListener(RoomEvent.Timeline, handleTimelineEvent); - }; - }, [mx, handleTimelineEvent]); + const dm = callInfo ? directs.has(callInfo.roomId) : false; + const startCall = useCallStart(); + // + // const handleSync = useCallback( + // async (syncValue: SyncValue) => { + // // only process rtc notification reference events. + // // we do not want to wait to decrypt all events. + // if (event.getRelation()?.rel_type !== RelationType.Reference) return; + // if (room?.isCallRoom()) return; + // + // if (event.isEncrypted()) { + // if (!event.isBeingDecrypted()) { + // await event.attemptDecryption(mx.getCrypto() as CryptoBackend); + // } + // await event.getDecryptionPromise(); + // } + // + // if ( + // !room || + // event.getType() !== EventType.RTCNotification || + // event.getSender() === mx.getSafeUserId() || + // !data.liveEvent + // ) { + // return; + // } + // + // const sender = event.getSender(); + // const content = event.getContent(); + // const senderTs = + // content.sender_ts - event.getTs() > 20000 ? event.getTs() : content.sender_ts; + // const lifetime = Math.min(content.lifetime, 120000); + // const notificationType = content.notification_type; + // const relation = + // event.getRelation()?.rel_type === RelationType.Reference ? event.getRelation() : undefined; + // const refEventId = relation?.event_id; + // + // const mention = + // content['m.mentions']?.room || + // content['m.mentions']?.user_ids?.includes(mx.getSafeUserId()); + // if (!sender || !refEventId || !mention || Date.now() >= senderTs + lifetime) { + // return; + // } + // + // const powerLevelsEvent = getStateEvent(room, StateEvent.RoomPowerLevels); + // const powerLevels = getPowersLevelFromMatrixEvent(powerLevelsEvent); + // const creators = getRoomCreatorsForRoomId(mx, room.roomId); + // const permissions = getRoomPermissionsAPI(creators, powerLevels); + // + // const hasCallPermission = permissions.stateEvent( + // StateEvent.GroupCallMemberPrefix, + // mx.getSafeUserId() + // ); + // if (!hasCallPermission) return; + // + // const info: IncomingCallInfo = { + // room, + // sender, + // senderTs, + // lifetime, + // intent: + // 'm.call.intent' in content && typeof content['m.call.intent'] === 'string' + // ? content['m.call.intent'] + // : undefined, + // notificationType, + // refEventId, + // }; + // + // setCallInfo(info); + // }, + // [mx] + // ); + // + // useEffect(() => { + // const unsub = ct.sync.observable.subscribe(handleSync) + // return () => { + // unsub() + // } + // }, [ct, handleSync]); const handleIgnore = useCallback(() => { setCallInfo(undefined); }, []); const handleReject = useCallback( - (room: Room, eventId: string) => { - mx.sendEvent(room.roomId, EventType.RTCDecline, { + async (roomId: string, eventId: string) => { + setCallInfo(undefined); + const content = { 'm.relates_to': { rel_type: RelationType.Reference, event_id: eventId, }, - }); - setCallInfo(undefined); + }; + + let encContent: object | undefined; + if (ct.encryptedRooms.read.has(roomId)) { + await ensureEncryptionSession(ct, roomId); + + const encStr = await ct.machine.roomEncryption.encryptMessageEvent( + roomId, + CallEventType.RTCDecline, + content + ); + encContent = JSON.parse(encStr); + } + + ct.requestAuthJSON( + putSend({ + pathParams: { + roomId, + eventType: encContent ? MessageEvent.RoomMessageEncrypted : CallEventType.RTCDecline, + txnId: ct.getTxnId(), + }, + body: encContent ?? content, + }) + ); }, - [mx] + [ct] ); const handleAnswer = useCallback( - (room: Room, video: boolean) => { - startCall(room, { microphone: true, video, sound: true }); + (roomId: string, video: boolean) => { + startCall(roomId, { microphone: true, video, sound: true }); setCallInfo(undefined); - navigateRoom(room.roomId); + navigateRoom(roomId); }, [startCall, navigateRoom] ); - if (callInfo && callEmbed?.roomId === callInfo.room.roomId) { + if (callInfo && callEmbed?.roomId === callInfo.roomId) { return null; } return !joined && callInfo ? ( diff --git a/src/app/features/call-status/CallRoomName.tsx b/src/app/features/call-status/CallRoomName.tsx index 8b65528a..7a56a8ae 100644 --- a/src/app/features/call-status/CallRoomName.tsx +++ b/src/app/features/call-status/CallRoomName.tsx @@ -1,35 +1,43 @@ import React from 'react'; -import { Room } from 'matrix-js-sdk'; import { Chip, Icon, Icons, Text } from 'folds'; import { useAtomValue } from 'jotai'; -import { useRoomName } from '../../hooks/useRoomMeta'; import { RoomIcon } from '../../components/room-avatar'; import { roomToParentsAtom } from '../../state/room/roomToParents'; import { getAllParents, guessPerfectParent } from '../../utils/room'; import { useOrphanSpaces } from '../../state/hooks/roomList'; -import { useMatrixClient } from '../../hooks/useMatrixClient'; import { allRoomsAtom } from '../../state/room-list/roomList'; import { mDirectAtom } from '../../state/mDirectList'; -import { useAllJoinedRoomsSet, useGetRoom } from '../../hooks/useGetRoom'; import { useRoomNavigate } from '../../hooks/useRoomNavigate'; +import { useClient } from '../../hooks/useClient'; +import { useGetRoomName } from '../../hooks/storage/useGetRoomName'; +import { useRoomHeroes } from '../../hooks/storage/useRoomHeros'; +import { useObservableValue } from '../../hooks/useObservableValue'; type CallRoomNameProps = { - room: Room; + roomId: string; }; -export function CallRoomName({ room }: CallRoomNameProps) { - const mx = useMatrixClient(); - const name = useRoomName(room); +export function CallRoomName({ roomId }: CallRoomNameProps) { + const ct = useClient(); + const roomToJoinRule = useObservableValue(ct.roomToJoinRule); + const getRoomHeroes = useRoomHeroes(); + const getRoomName = useGetRoomName(getRoomHeroes); + const name = getRoomName(roomId); + const roomToParents = useAtomValue(roomToParentsAtom); - const orphanSpaces = useOrphanSpaces(mx, allRoomsAtom, roomToParents); + const orphanSpaces = useOrphanSpaces(ct, allRoomsAtom, roomToParents); const mDirects = useAtomValue(mDirectAtom); - const dm = mDirects.has(room.roomId); + const dm = mDirects.has(roomId); - const allRoomsSet = useAllJoinedRoomsSet(); - const getRoom = useGetRoom(allRoomsSet); - - const allParents = getAllParents(roomToParents, room.roomId); + const allParents = getAllParents(roomToParents, roomId); const orphanParents = allParents && orphanSpaces.filter((o) => allParents.has(o)); - const perfectOrphanParent = orphanParents && guessPerfectParent(mx, room.roomId, orphanParents); + const perfectOrphanParent = + orphanParents && + guessPerfectParent( + roomId, + orphanParents, + (rId) => ct.roomToCreators.read.get(rId), + () => undefined + ); const { navigateRoom } = useRoomNavigate(); @@ -41,16 +49,21 @@ export function CallRoomName({ room }: CallRoomNameProps) { dm ? ( ) : ( - + ) } - onClick={() => navigateRoom(room.roomId)} + onClick={() => navigateRoom(roomId)} > {name} {!dm && perfectOrphanParent && ( - {' •'} {getRoom(perfectOrphanParent)?.name ?? perfectOrphanParent} + {' •'} {getRoomName(perfectOrphanParent) ?? perfectOrphanParent} )} diff --git a/src/app/features/call-status/CallStatus.tsx b/src/app/features/call-status/CallStatus.tsx index 58b8d399..404b56e5 100644 --- a/src/app/features/call-status/CallStatus.tsx +++ b/src/app/features/call-status/CallStatus.tsx @@ -1,4 +1,4 @@ -import React from 'react'; +import React, { useMemo } from 'react'; import { Box, Spinner } from 'folds'; import classNames from 'classnames'; import { LiveChip } from './LiveChip'; @@ -15,15 +15,19 @@ import { useCallEmbed, useCallJoined } from '../../hooks/useCallEmbed'; import { useCallSpeakers } from '../../hooks/useCallSpeakers'; import { MemberSpeaking } from './MemberSpeaking'; import { useSelectedRoom } from '../../hooks/router/useSelectedRoom'; +import { useRoomMembers } from '../../hooks/useRoomMembers'; type CallStatusProps = { callEmbed: CallEmbed; }; export function CallStatus({ callEmbed }: CallStatusProps) { - const { room } = callEmbed; + const { roomId } = callEmbed; - const callSession = useCallSession(room); + const callSession = useCallSession(roomId); const callMembers = useCallMembers(callSession); + const users = useMemo(() => Array.from(new Set(callMembers.map((m) => m.userId))), [callMembers]); + const membersAPI = useRoomMembers(roomId, users); + const screenSize = useScreenSize(); const callJoined = useCallJoined(callEmbed); const speakers = useCallSpeakers(callEmbed); @@ -43,7 +47,7 @@ export function CallStatus({ callEmbed }: CallStatusProps) { {memberVisible ? ( - + ) : ( @@ -51,12 +55,12 @@ export function CallStatus({ callEmbed }: CallStatusProps) { {!compact && ( <> - + {speakers.size > 0 && ( <> - + )} @@ -64,7 +68,12 @@ export function CallStatus({ callEmbed }: CallStatusProps) { {memberVisible && ( - + )} @@ -72,7 +81,7 @@ export function CallStatus({ callEmbed }: CallStatusProps) { {compact && ( - + )} diff --git a/src/app/features/call-status/LiveChip.tsx b/src/app/features/call-status/LiveChip.tsx index 32df3289..6769a1fd 100644 --- a/src/app/features/call-status/LiveChip.tsx +++ b/src/app/features/call-status/LiveChip.tsx @@ -15,26 +15,25 @@ import { Text, toRem, } from 'folds'; -import { CallMembership } from 'matrix-js-sdk/lib/matrixrtc/CallMembership'; import FocusTrap from 'focus-trap-react'; -import { Room } from 'matrix-js-sdk'; import * as css from './styles.css'; import { stopPropagation } from '../../utils/keyboard'; -import { getMemberAvatarMxc, getMemberDisplayName } from '../../utils/room'; -import { getMxIdLocalPart, mxcUrlToHttp } from '../../utils/matrix'; -import { useMatrixClient } from '../../hooks/useMatrixClient'; import { useMediaAuthentication } from '../../hooks/useMediaAuthentication'; import { UserAvatar } from '../../components/user-avatar'; import { useOpenUserRoomProfile } from '../../state/hooks/userRoomProfile'; import { getMouseEventCords } from '../../utils/dom'; +import { useClient } from '../../hooks/useClient'; +import { RoomMembersAPI } from '../../hooks/useRoomMembers'; +import { mxcToHttp } from '../../../client/mxcToHttp'; type LiveChipProps = { - room: Room; - members: CallMembership[]; + roomId: string; + users: string[]; count: number; + membersAPI: RoomMembersAPI; }; -export function LiveChip({ count, room, members }: LiveChipProps) { - const mx = useMatrixClient(); +export function LiveChip({ count, roomId, users, membersAPI }: LiveChipProps) { + const ct = useClient(); const useAuthentication = useMediaAuthentication(); const openUserProfile = useOpenUserRoomProfile(); @@ -70,26 +69,27 @@ export function LiveChip({ count, room, members }: LiveChipProps) { - {members.map((callMember) => { - const userId = callMember.sender; + {users.map((userId) => { if (!userId) return null; - const name = - getMemberDisplayName(room, userId) ?? getMxIdLocalPart(userId) ?? userId; - const avatarMxc = getMemberAvatarMxc(room, userId); + const name = membersAPI.getName(userId); + const avatarMxc = membersAPI.getAvatar(userId); const avatarUrl = avatarMxc - ? mxcUrlToHttp(mx, avatarMxc, useAuthentication, 96, 96) ?? undefined + ? mxcToHttp(ct.session.baseUrl, avatarMxc, useAuthentication).thumbnail({ + width: 96, + height: 96, + }) : undefined; return ( openUserProfile( - room.roomId, + roomId, undefined, userId, getMouseEventCords(evt.nativeEvent), diff --git a/src/app/features/call-status/MemberGlance.tsx b/src/app/features/call-status/MemberGlance.tsx index dabc440f..39ce39a5 100644 --- a/src/app/features/call-status/MemberGlance.tsx +++ b/src/app/features/call-status/MemberGlance.tsx @@ -1,59 +1,54 @@ import { Box, config, Icon, Icons, Text } from 'folds'; -import { CallMembership } from 'matrix-js-sdk/lib/matrixrtc/CallMembership'; import React from 'react'; -import { Room } from 'matrix-js-sdk'; import { UserAvatar } from '../../components/user-avatar'; -import { getMemberAvatarMxc, getMemberDisplayName } from '../../utils/room'; -import { getMxIdLocalPart, mxcUrlToHttp } from '../../utils/matrix'; -import { useMatrixClient } from '../../hooks/useMatrixClient'; import { useMediaAuthentication } from '../../hooks/useMediaAuthentication'; import { StackedAvatar } from '../../components/stacked-avatar'; import { useOpenUserRoomProfile } from '../../state/hooks/userRoomProfile'; import { getMouseEventCords } from '../../utils/dom'; import * as css from './styles.css'; +import { RoomMembersAPI } from '../../hooks/useRoomMembers'; +import { useClient } from '../../hooks/useClient'; +import { mxcToHttp } from '../../../client/mxcToHttp'; type MemberGlanceProps = { - room: Room; - members: CallMembership[]; + roomId: string; + users: string[]; + membersAPI: RoomMembersAPI; speakers: Set; max?: number; }; -export function MemberGlance({ room, members, speakers, max = 6 }: MemberGlanceProps) { - const mx = useMatrixClient(); +export function MemberGlance({ roomId, users, membersAPI, speakers, max = 6 }: MemberGlanceProps) { + const ct = useClient(); const useAuthentication = useMediaAuthentication(); const openUserProfile = useOpenUserRoomProfile(); - const visibleMembers = members.slice(0, max); - const remainingCount = max && members.length > max ? members.length - max : 0; + const visibleMembers = users.slice(0, max); + const remainingCount = max && users.length > max ? users.length - max : 0; return ( - {visibleMembers.map((callMember) => { - const { userId } = callMember; + {visibleMembers.map((userId) => { if (!userId) return null; - const name = getMemberDisplayName(room, userId) ?? getMxIdLocalPart(userId) ?? userId; - const avatarMxc = getMemberAvatarMxc(room, userId); + const name = membersAPI.getName(userId); + const avatarMxc = membersAPI.getAvatar(userId); const avatarUrl = avatarMxc - ? mxcUrlToHttp(mx, avatarMxc, useAuthentication, 96, 96) ?? undefined + ? mxcToHttp(ct.session.baseUrl, avatarMxc, useAuthentication).thumbnail({ + width: 96, + height: 96, + }) : undefined; return ( - openUserProfile( - room.roomId, - undefined, - userId, - getMouseEventCords(evt.nativeEvent), - 'Top' - ) + openUserProfile(roomId, undefined, userId, getMouseEventCords(evt.nativeEvent), 'Top') } > ; }; -export function MemberSpeaking({ room, speakers }: MemberSpeakingProps) { - const speakingNames = Array.from(speakers).map( - (userId) => getMemberDisplayName(room, userId) ?? getMxIdLocalPart(userId) ?? userId - ); +export function MemberSpeaking({ membersAPI, speakers }: MemberSpeakingProps) { + const speakingNames = Array.from(speakers).map((userId) => membersAPI.getName(userId)); + return ( diff --git a/src/app/features/call/CallMemberCard.tsx b/src/app/features/call/CallMemberCard.tsx index 8c37160b..c4292225 100644 --- a/src/app/features/call/CallMemberCard.tsx +++ b/src/app/features/call/CallMemberCard.tsx @@ -1,53 +1,51 @@ -import { CallMembership } from 'matrix-js-sdk/lib/matrixrtc/CallMembership'; import React, { useState } from 'react'; import { Avatar, Box, Icon, Icons, Text } from 'folds'; -import { useMatrixClient } from '../../hooks/useMatrixClient'; import { useMediaAuthentication } from '../../hooks/useMediaAuthentication'; import { useOpenUserRoomProfile } from '../../state/hooks/userRoomProfile'; import { SequenceCard } from '../../components/sequence-card'; -import { getMemberAvatarMxc, getMemberDisplayName } from '../../utils/room'; import { useRoom } from '../../hooks/useRoom'; -import { getMxIdLocalPart, mxcUrlToHttp } from '../../utils/matrix'; import { UserAvatar } from '../../components/user-avatar'; import { getMouseEventCords } from '../../utils/dom'; import * as css from './styles.css'; +import { CallMember } from '../../../client/call'; +import { useClient } from '../../hooks/useClient'; +import { useRoomMembersAPI } from '../../hooks/useRoomMembers'; +import { mxcToHttp } from '../../../client/mxcToHttp'; type CallMemberCardProps = { - member: CallMembership; + member: CallMember; }; export function CallMemberCard({ member }: CallMemberCardProps) { - const mx = useMatrixClient(); + const ct = useClient(); const useAuthentication = useMediaAuthentication(); - const room = useRoom(); + const roomId = useRoom(); + const membersAPI = useRoomMembersAPI(); const openUserProfile = useOpenUserRoomProfile(); const { userId } = member; if (!userId) return null; - const name = getMemberDisplayName(room, userId) ?? getMxIdLocalPart(userId) ?? userId; - const avatarMxc = getMemberAvatarMxc(room, userId); + const name = membersAPI.getName(member.userId); + const avatarMxc = membersAPI.getAvatar(member.userId); const avatarUrl = avatarMxc - ? mxcUrlToHttp(mx, avatarMxc, useAuthentication, 96, 96) ?? undefined + ? mxcToHttp(ct.session.baseUrl, avatarMxc, useAuthentication).thumbnail({ + width: 96, + height: 96, + }) : undefined; - const audioOnly = member.callIntent === 'audio'; + const audioOnly = member.intent === 'audio'; return ( - openUserProfile( - room.roomId, - undefined, - userId, - getMouseEventCords(evt.nativeEvent), - 'Right' - ) + openUserProfile(roomId, undefined, userId, getMouseEventCords(evt.nativeEvent), 'Right') } > @@ -70,13 +68,7 @@ export function CallMemberCard({ member }: CallMemberCardProps) { ); } -export function CallMemberRenderer({ - members, - max = 4, -}: { - members: CallMembership[]; - max?: number; -}) { +export function CallMemberRenderer({ members, max = 4 }: { members: CallMember[]; max?: number }) { const [viewMore, setViewMore] = useState(false); const truncatedMembers = viewMore ? members : members.slice(0, 4); @@ -85,7 +77,7 @@ export function CallMemberRenderer({ return ( <> {truncatedMembers.map((member) => ( - + ))} {members.length > max && ( Array.from(new Set(callMembers.map((m) => m.userId))), [callMembers]); + const hasParticipant = callMembers.length > 0; const callEmbed = useCallEmbed(); - const inOtherCall = callEmbed && callEmbed.roomId !== room.roomId; + const inOtherCall = callEmbed && callEmbed.roomId !== roomId; const canJoin = hasPermission && livekitSupported && rtcSupported; @@ -109,7 +109,7 @@ function CallPrescreen() { - {callMembers.length} Live + {users.length} Live @@ -151,14 +151,14 @@ function CallJoined({ joined, containerRef }: CallJoinedProps) { } export function CallView() { - const room = useRoom(); + const roomId = useRoom(); const callContainerRef = useRef(null); useCallEmbedPlacementSync(callContainerRef); const callEmbed = useCallEmbed(); const callJoined = useCallJoined(callEmbed); - const currentJoined = callEmbed?.roomId === room.roomId && callJoined; + const currentJoined = callEmbed?.roomId === roomId && callJoined; return ( toggleMicrophone(), [toggleMicrophone]); const handleVideoToggle = useCallback(async () => toggleVideo(), [toggleVideo]); @@ -53,7 +51,7 @@ export function PrescreenControls({ canJoin }: PrescreenControlsProps) { + + + + + ); + } +); + +function CallButton() { + const roomId = useRoom(); + + const callEmbed = useCallEmbed(); + const startCall = useCallStart(); + const callStarted = callEmbed && callEmbed.roomId === roomId; + const inAnotherCall = callEmbed && !callStarted; + const [menuAnchor, setMenuAnchor] = useState(); + + const handleOpenMenu: MouseEventHandler = (evt) => { + setMenuAnchor(evt.currentTarget.getBoundingClientRect()); + }; + + return ( + <> + + {inAnotherCall ? ( + Already in another call — End the current call to join! + ) : ( + Call + )} + + } + > + {(triggerRef) => ( + { + evt.preventDefault(); + startCall(roomId, { + microphone: true, + video: true, + sound: true, + }); + }} + disabled={inAnotherCall || callStarted} + aria-pressed={!!menuAnchor} + > + + + )} + + setMenuAnchor(undefined), + clickOutsideDeactivates: true, + isKeyForward: (evt: KeyboardEvent) => evt.key === 'ArrowDown', + isKeyBackward: (evt: KeyboardEvent) => evt.key === 'ArrowUp', + escapeDeactivates: stopPropagation, + }} + > + startCall(roomId, { microphone: true, video: true, sound: true })} + onVoiceCall={() => startCall(roomId, { microphone: true, video: false, sound: true })} + requestClose={() => setMenuAnchor(undefined)} + /> + + } + /> + + ); +} + export function RoomViewHeader({ callView }: { callView?: boolean }) { const navigate = useNavigate(); const ct = useClient(); @@ -271,6 +402,16 @@ export function RoomViewHeader({ callView }: { callView?: boolean }) { const encryptedRooms = useObservableValue(ct.encryptedRooms); const roomToJoinRule = useObservableValue(ct.roomToJoinRule); + const creators = useRoomCreators(roomId); + const powerLevels = usePowerLevelsContext(); + const permissions = useRoomPermissions(creators, powerLevels); + const hasCallPermission = permissions.stateEvent( + StateEvent.GroupCallMemberPrefix, + ct.session.userId + ); + const livekitSupported = useLivekitSupport(); + const rtcSupported = webRTCSupported(); + const pinnedEvents = useRoomPinnedEvents(roomId); const getRoomHeroes = useRoomHeroes(); @@ -474,6 +615,11 @@ export function RoomViewHeader({ callView }: { callView?: boolean }) { } /> + {!isCallRoom(ct.roomToType.read.get(roomId)) && + livekitSupported && + rtcSupported && + hasCallPermission && } + {screenSize === ScreenSize.Desktop && ( { - const ct = useClient(); - const roomId = useRoom(); - - const { memberEventsResult, getMembership } = useRoomMembersAPI(); - - const prepareEncryptionSession = useCallback(async () => { - if (!ct.encryptedRooms.read.has(roomId)) { - return; - } - const historyVisibility = ct.roomToHistoryVisibility.read.get(roomId) ?? 'shared'; - const encryptForInvited = historyVisibility !== 'joined'; - - // TODO: make sure all members are loaded. - const targetMembers = memberEventsResult.value - ?.map((e) => e.userId) - .filter((userId) => { - const membership = getMembership(userId); - return membership === 'join' || (membership === 'invite' && encryptForInvited); - }); - - if (!targetMembers) return; - - await ct.machine.userTracker.track(targetMembers); - await ct.machine.outgoingRequests.processAll(); - await ct.machine.outgoingRequests.claimKeys(targetMembers); - - const roomSettings = await ct.machine.roomEncryption.getSettings(roomId); - const encryptionEventContent = ct.state.join.encryption.read.find((ev) => ev.roomId === roomId) - ?.event.content; - - const encryptionSettings: EncryptionSettings = buildEncryptionSettings( - historyVisibility, - roomSettings, - encryptionEventContent - ); - - await ct.machine.roomKeys.shareRoomKey(roomId, targetMembers, encryptionSettings); - }, [ct, roomId, memberEventsResult, getMembership]); - - return prepareEncryptionSession; -}; diff --git a/src/app/features/room/hooks/useSendEditedMessage.ts b/src/app/features/room/hooks/useSendEditedMessage.ts index 80c20c7b..f395613f 100644 --- a/src/app/features/room/hooks/useSendEditedMessage.ts +++ b/src/app/features/room/hooks/useSendEditedMessage.ts @@ -3,7 +3,7 @@ import { Editor } from 'slate'; import { MessageEventWithoutRoomId } from 'neurone/lib/api/types'; import { useClient } from '../../../hooks/useClient'; import { useRoom } from '../../../hooks/useRoom'; -import { useEnsureRoomEncryptionSession } from './useEnsureRoomEncryptionSession'; +import { useEnsureRoomEncryptionSession } from '../../../hooks/useEnsureRoomEncryptionSession'; import { EventSenderIdle, LocalEvent, diff --git a/src/app/features/room/hooks/useSendMessage.ts b/src/app/features/room/hooks/useSendMessage.ts index 48fc1f98..136afd74 100644 --- a/src/app/features/room/hooks/useSendMessage.ts +++ b/src/app/features/room/hooks/useSendMessage.ts @@ -36,7 +36,7 @@ import { RelationType, StickerContent, } from '../../../../types/matrix/common'; -import { useEnsureRoomEncryptionSession } from './useEnsureRoomEncryptionSession'; +import { useEnsureRoomEncryptionSession } from '../../../hooks/useEnsureRoomEncryptionSession'; import { LocalMessage } from '../../../../client/room/local-event'; import { useTimelineView } from './useTimeline'; diff --git a/src/app/features/room/thread-view/ThreadView.tsx b/src/app/features/room/thread-view/ThreadView.tsx index 3a8a0c07..7921b74b 100644 --- a/src/app/features/room/thread-view/ThreadView.tsx +++ b/src/app/features/room/thread-view/ThreadView.tsx @@ -1,11 +1,7 @@ import React, { MutableRefObject, useRef } from 'react'; -import classNames from 'classnames'; -import { Box, config, Icon, IconButton, Icons, Text, Tooltip, TooltipProvider } from 'folds'; -import FocusTrap from 'focus-trap-react'; -import * as css from './styles.css'; +import { Box, config, Icon, IconButton, Icons, Text, Tooltip, TooltipProvider, toRem } from 'folds'; import { ThreadTimeline } from '../../../../client/room/ThreadTimeline'; import { Page, PageHeader } from '../../../components/page'; -import { ScreenSize, useScreenSizeContext } from '../../../hooks/useScreenSize'; import { useEditor } from '../../../components/editor'; import { RoomInput } from '../RoomInput'; import { TimelineView } from './TimelineView'; @@ -13,6 +9,7 @@ import { useEditorAutoFocus, useThreadFocus } from '../hooks/useEditorAutoFocus' import { useSetting } from '../../../state/hooks/settings'; import { FollowingMembersPlaceholder } from '../components'; import { ThreadViewFollowing } from './ThreadViewFollowing'; +import { ScreenSize, useScreenSizeContext } from '../../../hooks/useScreenSize'; type ThreadViewProps = { threadTimeline: ThreadTimeline; @@ -22,7 +19,6 @@ type ThreadViewProps = { export function ThreadView({ threadTimeline, threadFocusRef, requestClose }: ThreadViewProps) { const screenSize = useScreenSizeContext(); - const floating = screenSize !== ScreenSize.Desktop; const editor = useEditor(); const threadInputRef = useRef(null); const threadViewRef = useRef(null); @@ -32,69 +28,62 @@ export function ThreadView({ threadTimeline, threadFocusRef, requestClose }: Thr useEditorAutoFocus(editor, threadFocusRef, true); return ( - - - - - - - Thread - - - - - Close - - } - > - {(triggerRef) => ( - - - - )} - - + + + + + Thread + + + + + Close + + } + > + {(triggerRef) => ( + + + + )} + - - - - -
- -
- {hideActivity ? ( - - ) : ( - - )}
-
-
+ + + + +
+ +
+ {hideActivity ? ( + + ) : ( + + )} +
+ ); } diff --git a/src/app/features/room/thread-view/styles.css.ts b/src/app/features/room/thread-view/styles.css.ts index 9a0867e9..7aa734c0 100644 --- a/src/app/features/room/thread-view/styles.css.ts +++ b/src/app/features/room/thread-view/styles.css.ts @@ -1,27 +1,6 @@ import { keyframes, style } from '@vanilla-extract/css'; -import { config, toRem } from 'folds'; import { ContainerColor } from '../../../styles/ContainerColor.css'; -export const ThreadView = style({ - // width: toRem(456), - width: toRem(496), - flexShrink: 0, - flexGrow: 0, -}); - -export const ThreadViewFloating = style({ - position: 'absolute', - right: 0, - top: 0, - bottom: 0, - zIndex: 1, - - maxWidth: toRem(496), - flexShrink: 1, - width: '100vw', - boxShadow: config.shadow.E400, -}); - export const ZigZagLine = style([ ContainerColor({ variant: 'Background' }), { diff --git a/src/app/hooks/useAllJoinedRoomsSet.ts b/src/app/hooks/useAllJoinedRoomsSet.ts new file mode 100644 index 00000000..09a451f9 --- /dev/null +++ b/src/app/hooks/useAllJoinedRoomsSet.ts @@ -0,0 +1,10 @@ +import { useAtomValue } from 'jotai'; +import { useMemo } from 'react'; +import { allRoomsAtom } from '../state/room-list/roomList'; + +export const useAllJoinedRoomsSet = () => { + const allRooms = useAtomValue(allRoomsAtom); + const allJoinedRooms = useMemo(() => new Set(allRooms), [allRooms]); + + return allJoinedRooms; +}; diff --git a/src/app/hooks/useCall.ts b/src/app/hooks/useCall.ts index 817e12fc..8d1ec8bf 100644 --- a/src/app/hooks/useCall.ts +++ b/src/app/hooks/useCall.ts @@ -1,60 +1,28 @@ -import { Room } from 'matrix-js-sdk'; -import { - MatrixRTCSession, - MatrixRTCSessionEvent, - MatrixRTCSessionEventHandlerMap, -} from 'matrix-js-sdk/lib/matrixrtc/MatrixRTCSession'; -import { CallMembership } from 'matrix-js-sdk/lib/matrixrtc/CallMembership'; -import { useEffect, useState } from 'react'; -import { MatrixRTCSessionManagerEvents } from 'matrix-js-sdk/lib/matrixrtc/MatrixRTCSessionManager'; -import { useMatrixClient } from './useMatrixClient'; +import { useEffect } from 'react'; +import { useClient } from './useClient'; +import { CallMember, CallSession } from '../../client/call'; +import { useObservableValue } from './useObservableValue'; -export const useCallSession = (room: Room): MatrixRTCSession => { - const mx = useMatrixClient(); +export const useCallSession = (roomId: string): CallSession => { + const ct = useClient(); - const [session, setSession] = useState(mx.matrixRTC.getRoomSession(room)); - - useEffect(() => { - const start = (roomId: string) => { - if (roomId !== room.roomId) return; - setSession(mx.matrixRTC.getRoomSession(room)); - }; - const end = (roomId: string) => { - if (roomId !== room.roomId) return; - setSession(mx.matrixRTC.getRoomSession(room)); - }; - mx.matrixRTC.on(MatrixRTCSessionManagerEvents.SessionStarted, start); - mx.matrixRTC.on(MatrixRTCSessionManagerEvents.SessionEnded, end); - return () => { - mx.matrixRTC.off(MatrixRTCSessionManagerEvents.SessionStarted, start); - mx.matrixRTC.off(MatrixRTCSessionManagerEvents.SessionEnded, end); - }; - }, [mx, room]); - - return session; + return ct.callSessions.getSession(roomId); }; export const useCallMembersChange = ( - session: MatrixRTCSession, - callback: (members: CallMembership[]) => void + session: CallSession, + callback: (members: CallMember[]) => void ): void => { useEffect(() => { - const handleMembershipsChange: MatrixRTCSessionEventHandlerMap[MatrixRTCSessionEvent.MembershipsChanged] = - (oldestMembership, newMemberships) => { - callback(newMemberships); - }; - - session.on(MatrixRTCSessionEvent.MembershipsChanged, handleMembershipsChange); + const unsub = session.members.subscribe(callback); return () => { - session.removeListener(MatrixRTCSessionEvent.MembershipsChanged, handleMembershipsChange); + unsub(); }; }, [session, callback]); }; -export const useCallMembers = (session: MatrixRTCSession): CallMembership[] => { - const [memberships, setMemberships] = useState(session.memberships); +export const useCallMembers = (session: CallSession): CallMember[] => { + const members = useObservableValue(session.members); - useCallMembersChange(session, setMemberships); - - return memberships; + return members; }; diff --git a/src/app/hooks/useCallEmbed.ts b/src/app/hooks/useCallEmbed.ts index af6f47b4..d02fadbe 100644 --- a/src/app/hooks/useCallEmbed.ts +++ b/src/app/hooks/useCallEmbed.ts @@ -1,5 +1,4 @@ import { createContext, RefObject, useCallback, useContext, useEffect, useState } from 'react'; -import { MatrixClient, Room } from 'matrix-js-sdk'; import { useSetAtom } from 'jotai'; import { CallEmbed, @@ -7,13 +6,14 @@ import { ElementWidgetActions, useClientWidgetApiEvent, } from '../plugins/call'; -import { useMatrixClient } from './useMatrixClient'; import { ThemeKind, useTheme } from './useTheme'; import { callEmbedAtom } from '../state/callEmbed'; import { useResizeObserver } from './useResizeObserver'; import { CallControlState } from '../plugins/call/CallControlState'; import { useCallMembersChange, useCallSession } from './useCall'; import { CallPreferences } from '../state/callPreferences'; +import { ClientContext } from '../../client/context'; +import { useClient } from './useClient'; const CallEmbedContext = createContext(undefined); @@ -36,42 +36,49 @@ export const useCallEmbedRef = (): RefObject => { }; export const createCallEmbed = ( - mx: MatrixClient, - room: Room, + ct: ClientContext, + roomId: string, dm: boolean, themeKind: ElementCallThemeKind, container: HTMLElement, pref?: CallPreferences ): CallEmbed => { - const rtcSession = mx.matrixRTC.getRoomSession(room); - const ongoing = rtcSession.memberships.length > 0; + const callSession = ct.callSessions.getSession(roomId); + const ongoing = callSession.members.read.length > 0; const intent = CallEmbed.getIntent(dm, ongoing, pref?.video); - const widget = CallEmbed.getWidget(mx, room, intent, themeKind); + const widget = CallEmbed.getWidget(ct, roomId, intent, themeKind); const controlState = pref && new CallControlState(pref.microphone, pref.video, pref.sound); - const embed = new CallEmbed(mx, room, widget, container, controlState); + const embed = new CallEmbed(ct, roomId, widget, container, controlState); return embed; }; -export const useCallStart = (dm = false) => { - const mx = useMatrixClient(); +export const useCallStart = () => { + const ct = useClient(); const theme = useTheme(); const setCallEmbed = useSetAtom(callEmbedAtom); const callEmbedRef = useCallEmbedRef(); const startCall = useCallback( - (room: Room, pref?: CallPreferences) => { + (roomId: string, pref?: CallPreferences) => { const container = callEmbedRef.current; if (!container) { throw new Error('Failed to start call, No embed container element found!'); } - const callEmbed = createCallEmbed(mx, room, dm, theme.kind, container, pref); + const callEmbed = createCallEmbed( + ct, + roomId, + ct.directRooms.read.has(roomId), + theme.kind, + container, + pref + ); setCallEmbed(callEmbed); }, - [mx, dm, theme, setCallEmbed, callEmbedRef] + [ct, theme, setCallEmbed, callEmbedRef] ); return startCall; @@ -103,7 +110,7 @@ export const useCallHangupEvent = (embed: CallEmbed, callback: () => void) => { }; export const useCallMemberSoundSync = (embed: CallEmbed) => { - const callSession = useCallSession(embed.room); + const callSession = useCallSession(embed.roomId); useCallMembersChange( callSession, useCallback(() => embed.control.applySound(), [embed]) diff --git a/src/app/hooks/useCallSpeakers.ts b/src/app/hooks/useCallSpeakers.ts index 7f720074..a26dfbf3 100644 --- a/src/app/hooks/useCallSpeakers.ts +++ b/src/app/hooks/useCallSpeakers.ts @@ -7,7 +7,7 @@ import { useCallJoined } from './useCallEmbed'; export const useCallSpeakers = (callEmbed: CallEmbed): Set => { const [speakers, setSpeakers] = useState(new Set()); - const callSession = useCallSession(callEmbed.room); + const callSession = useCallSession(callEmbed.roomId); const callMembers = useCallMembers(callSession); const joined = useCallJoined(callEmbed); diff --git a/src/app/hooks/useEnsureRoomEncryptionSession.ts b/src/app/hooks/useEnsureRoomEncryptionSession.ts new file mode 100644 index 00000000..bace85fb --- /dev/null +++ b/src/app/hooks/useEnsureRoomEncryptionSession.ts @@ -0,0 +1,83 @@ +import { useCallback } from 'react'; +import { + EncryptionSettings, + HistoryVisibility, + RoomEncryptionSettings, +} from 'neurone/lib/crypto/machine'; +import { ObjectReader } from 'neurone/lib/utils'; +import { useClient } from './useClient'; +import { useRoom } from './useRoom'; +import { ClientContext } from '../../client/context'; +import { loadPendingMembers } from './usePendingMembers'; + +export function buildEncryptionSettings( + historyVisibility: HistoryVisibility, + roomSettings: RoomEncryptionSettings | undefined, + encryptionEventContent: object | undefined +): EncryptionSettings { + const rotationPeriodMs = ObjectReader.number(encryptionEventContent, 'rotation_period_ms'); + + const rotationPeriodMsgs = ObjectReader.number(encryptionEventContent, 'rotation_period_msgs'); + + const rotationMilliseconds = roomSettings?.sessionRotationPeriodMs ?? rotationPeriodMs; + + return { + algorithm: 'm.megolm.v1.aes-sha2', + historyVisibility, + encryptStateEvents: roomSettings?.encryptStateEvents, + // EncryptionSettings expects microseconds. + rotationPeriod: rotationMilliseconds === undefined ? undefined : rotationMilliseconds * 1_000, + rotationPeriodMessages: roomSettings?.sessionRotationPeriodMessages ?? rotationPeriodMsgs, + sharingStrategy: roomSettings?.onlyAllowTrustedDevices ? 'only-trusted-devices' : 'all-devices', + }; +} + +export const ensureEncryptionSession = async (ct: ClientContext, roomId: string) => { + if (!ct.encryptedRooms.read.has(roomId)) { + return; + } + const historyVisibility = ct.roomToHistoryVisibility.read.get(roomId) ?? 'shared'; + const encryptForInvited = historyVisibility !== 'joined'; + + const memberObjs = await ct.storage.joinRoomMember.read.queryIndex('roomId', roomId); + + if (ct.pendingMemberRooms.read.includes(roomId)) { + await loadPendingMembers(ct, roomId); + } + + const targetMembers = memberObjs + .filter((memberObj) => { + const membership = ObjectReader.string(memberObj.event.content, 'membership'); + return membership === 'join' || (membership === 'invite' && encryptForInvited); + }) + .map((memberObj) => memberObj.userId); + + if (!targetMembers) return; + + await ct.machine.userTracker.track(targetMembers); + await ct.machine.outgoingRequests.processAll(); + await ct.machine.outgoingRequests.claimKeys(targetMembers); + + const roomSettings = await ct.machine.roomEncryption.getSettings(roomId); + const encryptionEventContent = ct.state.join.encryption.read.find((ev) => ev.roomId === roomId) + ?.event.content; + + const encryptionSettings: EncryptionSettings = buildEncryptionSettings( + historyVisibility, + roomSettings, + encryptionEventContent + ); + + await ct.machine.roomKeys.shareRoomKey(roomId, targetMembers, encryptionSettings); +}; + +export const useEnsureRoomEncryptionSession = () => { + const ct = useClient(); + const roomId = useRoom(); + + const prepareEncryptionSession = useCallback(async () => { + await ensureEncryptionSession(ct, roomId); + }, [ct, roomId]); + + return prepareEncryptionSession; +}; diff --git a/src/app/hooks/useGetRoom.ts b/src/app/hooks/useGetRoom.ts deleted file mode 100644 index 05fcf8d3..00000000 --- a/src/app/hooks/useGetRoom.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { Room } from 'matrix-js-sdk'; -import { useAtomValue } from 'jotai'; -import { useCallback, useMemo } from 'react'; -import { allRoomsAtom } from '../state/room-list/roomList'; -import { useMatrixClient } from './useMatrixClient'; - -export const useAllJoinedRoomsSet = () => { - const allRooms = useAtomValue(allRoomsAtom); - const allJoinedRooms = useMemo(() => new Set(allRooms), [allRooms]); - - return allJoinedRooms; -}; - -export type GetRoomCallback = (roomId: string) => Room | undefined; -export const useGetRoom = (rooms: Set): GetRoomCallback => { - const mx = useMatrixClient(); - - const getRoom: GetRoomCallback = useCallback( - (rId: string) => { - if (rooms.has(rId)) { - return mx.getRoom(rId) ?? undefined; - } - return undefined; - }, - [mx, rooms] - ); - - return getRoom; -}; diff --git a/src/app/hooks/useImagePackRooms.ts b/src/app/hooks/useImagePackRooms.ts index 25636589..9e650efc 100644 --- a/src/app/hooks/useImagePackRooms.ts +++ b/src/app/hooks/useImagePackRooms.ts @@ -1,7 +1,7 @@ import { useAtomValue } from 'jotai'; import { useMemo } from 'react'; import { getAllParents } from '../utils/room'; -import { useAllJoinedRoomsSet } from './useGetRoom'; +import { useAllJoinedRoomsSet } from './useAllJoinedRoomsSet'; import { roomToParentsAtom } from '../state/room/roomToParents'; export const useImagePackRooms = (roomId: string): string[] => { diff --git a/src/app/hooks/usePendingMembers.ts b/src/app/hooks/usePendingMembers.ts index d533c8c9..883d112f 100644 --- a/src/app/hooks/usePendingMembers.ts +++ b/src/app/hooks/usePendingMembers.ts @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useMemo } from 'react'; +import { useEffect, useMemo } from 'react'; import { getMembers, GetMembersOK } from 'neurone/lib/api/matrix/client/v3/rooms'; import { parseMatrixResp, printMatrixError } from 'neurone/lib/matrix-request'; import { StateEventWithoutRoomId } from 'neurone/lib/api/types'; @@ -9,7 +9,7 @@ import { JoinRoomMemberObject, sessionValue, } from '../../client/storage/collections'; -import { useAlive } from './useAlive'; +import { ClientContext } from '../../client/context'; export const usePendingMembers = (roomId: string): boolean => { const ct = useClient(); @@ -22,53 +22,51 @@ export const usePendingMembers = (roomId: string): boolean => { return hasPendingMembers; }; +export const loadPendingMembers = async ( + ct: ClientContext, + roomId: string, + signal?: AbortSignal +) => { + const response = await ct.requestAuthJSON(getMembers({ pathParams: { roomId } }), signal); + const [err, result] = await parseMatrixResp(response); + if (err) { + throw new Error(printMatrixError(err)); + } + + const memberObjects: JoinRoomMemberObject[] = + result?.chunk.map((event) => { + const eventWithoutRoomId: StateEventWithoutRoomId = event; + if ('room_id' in eventWithoutRoomId) delete eventWithoutRoomId.room_id; + + const obj: JoinRoomMemberObject = { + id: getJoinRoomMemberId(roomId, eventWithoutRoomId.state_key), + roomId, + userId: eventWithoutRoomId.state_key, + event: eventWithoutRoomId, + }; + + return obj; + }) ?? []; + + await ct.storage.joinRoomMember.write.putMultiple(memberObjects); + await ct.storage.sessionValue.write.put({ + id: sessionValue.pendingMemberRooms.id, + value: ct.pendingMemberRooms.read.filter((rId) => rId !== roomId), + }); +}; + export const usePendingMemberLoader = (roomId: string, pending: boolean) => { const ct = useClient(); - const alive = useAlive(); - - const loadMembers = useCallback( - async (signal?: AbortSignal) => { - const response = await ct.requestAuthJSON(getMembers({ pathParams: { roomId } }), signal); - const [err, result] = await parseMatrixResp(response); - if (err) { - throw new Error(printMatrixError(err)); - } - - const memberObjects: JoinRoomMemberObject[] = - result?.chunk.map((event) => { - const eventWithoutRoomId: StateEventWithoutRoomId = event; - if ('room_id' in eventWithoutRoomId) delete eventWithoutRoomId.room_id; - - const obj: JoinRoomMemberObject = { - id: getJoinRoomMemberId(roomId, eventWithoutRoomId.state_key), - roomId, - userId: eventWithoutRoomId.state_key, - event: eventWithoutRoomId, - }; - - return obj; - }) ?? []; - - if (alive()) { - await ct.storage.joinRoomMember.write.putMultiple(memberObjects); - await ct.storage.sessionValue.write.put({ - id: sessionValue.pendingMemberRooms.id, - value: ct.pendingMemberRooms.read.filter((rId) => rId !== roomId), - }); - } - }, - [ct, roomId, alive] - ); useEffect(() => { const abortController = new AbortController(); if (pending) { - loadMembers(abortController.signal); + loadPendingMembers(ct, roomId, abortController.signal); } return () => { abortController.abort(); }; - }, [pending, loadMembers]); + }, [ct, pending, roomId]); }; diff --git a/src/app/hooks/usePowerLevels.ts b/src/app/hooks/usePowerLevels.ts index ef549c1f..71a334be 100644 --- a/src/app/hooks/usePowerLevels.ts +++ b/src/app/hooks/usePowerLevels.ts @@ -1,4 +1,3 @@ -import { MatrixEvent } from 'matrix-js-sdk'; import { createContext, useCallback, useContext, useMemo } from 'react'; import produce from 'immer'; import { PendingObservableResult } from 'neurone/lib/utils/observable'; @@ -57,14 +56,6 @@ const fillMissingPowers = (powerLevels: IPowerLevels): IPowerLevels => return draftPl; }); -export const getPowersLevelFromMatrixEvent = (mEvent?: MatrixEvent): IPowerLevels => { - const plContent = mEvent?.getContent(); - - const powerLevels = !plContent ? DEFAULT_POWER_LEVELS : fillMissingPowers(plContent); - - return powerLevels; -}; - export function usePowerLevelsState( roomId: string ): PendingObservableResult { diff --git a/src/app/hooks/useSpaceHierarchy.ts b/src/app/hooks/useSpaceHierarchy.ts index 61c659d3..c4e3866b 100644 --- a/src/app/hooks/useSpaceHierarchy.ts +++ b/src/app/hooks/useSpaceHierarchy.ts @@ -10,7 +10,7 @@ import { SortFunc, byOrderKey, byTsOldToNew, factoryRoomIdByActivity } from '../ import { ErrorCode } from '../cs-errorcode'; import { useObservableValue } from './useObservableValue'; import { useClient } from './useClient'; -import { useAllJoinedRoomsSet } from './useGetRoom'; +import { useAllJoinedRoomsSet } from './useAllJoinedRoomsSet'; import { spaceRoomsAtom } from '../state/spaceRooms'; import { rError } from '../../client/reader'; import { useRoomsLatestEvent } from './useRoomsLatestEvent'; diff --git a/src/app/pages/Router.tsx b/src/app/pages/Router.tsx index 72a34755..865cff08 100644 --- a/src/app/pages/Router.tsx +++ b/src/app/pages/Router.tsx @@ -83,15 +83,18 @@ export const createRouter = (clientConfig: ClientConfig, screenSize: ScreenSize) - - - - } - > - - + + + + + } + > + + + + diff --git a/src/app/pages/client/space/Space.tsx b/src/app/pages/client/space/Space.tsx index 2cb02bcf..d5d3c431 100644 --- a/src/app/pages/client/space/Space.tsx +++ b/src/app/pages/client/space/Space.tsx @@ -89,7 +89,7 @@ import { useRoomsTotalNotifications, } from '../../../state/hooks/roomNotifications'; import { getJoinRoomStateId } from '../../../../client/storage/collections'; -import { useAllJoinedRoomsSet } from '../../../hooks/useGetRoom'; +import { useAllJoinedRoomsSet } from '../../../hooks/useAllJoinedRoomsSet'; import { unreadRoomsAtom } from '../../../state/room/roomToUnread'; import { useRoomsUnread } from '../../../state/hooks/unread'; import { RoomAccessIndicator } from '../../../components/RoomAccessIndicator'; diff --git a/src/app/plugins/call/CallControl.ts b/src/app/plugins/call/CallControl.ts index 62f6966a..211280f4 100644 --- a/src/app/plugins/call/CallControl.ts +++ b/src/app/plugins/call/CallControl.ts @@ -1,5 +1,6 @@ import { ClientWidgetApi } from 'matrix-widget-api'; -import EventEmitter from 'events'; +import { ObservableValue } from 'neurone/lib/utils/observable'; +import { Disposable } from 'neurone/lib/utils/disposable'; import { CallControlState } from './CallControlState'; import { ElementMediaStateDetail, ElementMediaStatePayload, ElementWidgetActions } from './types'; @@ -7,16 +8,18 @@ export enum CallControlEvent { StateUpdate = 'state_update', } -export class CallControl extends EventEmitter implements CallControlState { - private state: CallControlState; +export class CallControl implements CallControlState, Disposable { + private readonly publishKey = crypto.randomUUID(); - private call: ClientWidgetApi; + public readonly state: ObservableValue; - private iframe: HTMLIFrameElement; + private readonly call: ClientWidgetApi; - private bodyMutationObserver: MutationObserver; + private readonly iframe: HTMLIFrameElement; - private controlMutationObserver: MutationObserver; + private readonly bodyMutationObserver: MutationObserver; + + private readonly controlMutationObserver: MutationObserver; private mediaStatePromiseResolver: undefined | (() => void); @@ -72,9 +75,7 @@ export class CallControl extends EventEmitter implements CallControlState { } constructor(state: CallControlState, call: ClientWidgetApi, iframe: HTMLIFrameElement) { - super(); - - this.state = state; + this.state = new ObservableValue(state, this.publishKey); this.call = call; this.iframe = iframe; @@ -82,28 +83,24 @@ export class CallControl extends EventEmitter implements CallControlState { this.controlMutationObserver = new MutationObserver(this.onControlMutation.bind(this)); } - public getState(): CallControlState { - return this.state; - } - public get microphone(): boolean { - return this.state.microphone; + return this.state.read.microphone; } public get video(): boolean { - return this.state.video; + return this.state.read.video; } public get sound(): boolean { - return this.state.sound; + return this.state.read.sound; } public get screenshare(): boolean { - return this.state.screenshare; + return this.state.read.screenshare; } public get spotlight(): boolean { - return this.state.spotlight; + return this.state.read.spotlight; } public async applyState() { @@ -112,7 +109,6 @@ export class CallControl extends EventEmitter implements CallControlState { video_enabled: this.video, }); this.setSound(this.sound); - this.emitStateUpdate(); } public startObserving() { @@ -124,10 +120,18 @@ export class CallControl extends EventEmitter implements CallControlState { }); this.onBodyMutation(); } - + private onBodyMutation() { if (!this.document) return; + const style = this.document.createElement('style'); + style.textContent = ` + body::before { + display: none !important; + } + `; + this.document.head.appendChild(style); + this.document.body.style.setProperty('background', 'none', 'important'); const controls = this.leaveButton?.parentElement?.parentElement; @@ -165,7 +169,7 @@ export class CallControl extends EventEmitter implements CallControlState { private async setMediaState(state: ElementMediaStatePayload) { const data = await this.call.transport.send(ElementWidgetActions.DeviceMute, state); - return new Promise(resolve => { + return new Promise((resolve) => { if (this.mediaStatePromiseResolver) { this.mediaStatePromiseResolver(); } @@ -195,8 +199,7 @@ export class CallControl extends EventEmitter implements CallControlState { this.spotlight ); - this.state = state; - this.emitStateUpdate(); + this.state.publish(this.publishKey, state); if (this.microphone && !this.sound) { this.toggleSound(); @@ -212,14 +215,10 @@ export class CallControl extends EventEmitter implements CallControlState { const screenshare: boolean = this.screenshareButton?.getAttribute('data-kind') === 'primary'; const spotlight: boolean = this.spotlightButton?.checked ?? false; - this.state = new CallControlState( - this.microphone, - this.video, - this.sound, - screenshare, - spotlight + this.state.publish( + this.publishKey, + new CallControlState(this.microphone, this.video, this.sound, screenshare, spotlight) ); - this.emitStateUpdate(); } public toggleMicrophone() { @@ -250,8 +249,7 @@ export class CallControl extends EventEmitter implements CallControlState { this.screenshare, this.spotlight ); - this.state = state; - this.emitStateUpdate(); + this.state.publish(this.publishKey, state); if (!this.sound && this.microphone) { this.toggleMicrophone(); @@ -282,8 +280,4 @@ export class CallControl extends EventEmitter implements CallControlState { this.bodyMutationObserver.disconnect(); this.controlMutationObserver.disconnect(); } - - private emitStateUpdate() { - this.emit(CallControlEvent.StateUpdate); - } } diff --git a/src/app/plugins/call/CallEmbed.ts b/src/app/plugins/call/CallEmbed.ts index 8d03bc8a..46c048d9 100644 --- a/src/app/plugins/call/CallEmbed.ts +++ b/src/app/plugins/call/CallEmbed.ts @@ -1,20 +1,18 @@ -import { - ClientEvent, - KnownMembership, - MatrixClient, - MatrixEvent, - MatrixEventEvent, - Room, - RoomStateEvent, -} from 'matrix-js-sdk'; import { ClientWidgetApi, - IRoomEvent, IWidget, Widget, WidgetApiToWidgetAction, WidgetDriver, } from 'matrix-widget-api'; +import { ObjectReader } from 'neurone/lib/utils'; +import { Disposable, Disposer } from 'neurone/lib/utils/disposable'; +import { + ProcessedToDeviceEvent, + ToDeviceDecrypted, + ToDeviceUTD, +} from 'neurone/lib/crypto/machine/events'; +import { ClientEventWithoutRoomId, ToDeviceEvent } from 'neurone/lib/api/types'; import { CallWidgetDriver } from './CallWidgetDriver'; import { trimTrailingSlash } from '../../utils/common'; import { @@ -25,27 +23,25 @@ import { } from './types'; import { CallControl } from './CallControl'; import { CallControlState } from './CallControlState'; +import { ClientContext } from '../../../client/context'; +import { RoomType } from '../../../types/matrix/room'; -export class CallEmbed { - private mx: MatrixClient; +export class CallEmbed implements Disposable { + private ct: ClientContext; + + public readonly roomId: string; public readonly call: ClientWidgetApi; public readonly iframe: HTMLIFrameElement; - public readonly room: Room; - public joined = false; public readonly control: CallControl; private readonly container: HTMLElement; - private readUpToMap: { [roomId: string]: string } = {}; // room ID to event ID - - private eventsToFeed = new WeakSet(); - - private readonly disposables: Array<() => void> = []; + private readonly disposer = new Disposer(); static getIntent(dm: boolean, ongoing: boolean, video?: boolean): ElementCallIntent { if (dm && ongoing) { @@ -80,21 +76,21 @@ export class CallEmbed { } static getWidget( - mx: MatrixClient, - room: Room, + ct: ClientContext, + roomId: string, intent: ElementCallIntent, themeKind: ElementCallThemeKind ): Widget { - const userId = mx.getSafeUserId(); - const deviceId = mx.getDeviceId() ?? ''; + const { userId, deviceId, baseUrl } = ct.session; + const clientOrigin = window.location.origin; const widgetId = 'call-embed'; const params = new URLSearchParams({ widgetId, parentUrl: clientOrigin, - baseUrl: mx.baseUrl, - roomId: room.roomId, + baseUrl, + roomId, userId, deviceId, intent, @@ -102,13 +98,14 @@ export class CallEmbed { skipLobby: 'true', confineToRoom: 'true', appPrompt: 'false', - perParticipantE2EE: room.hasEncryptionStateEvent().toString(), lang: 'en-EN', theme: themeKind, header: 'none', }); - if (!room.isCallRoom() && CallEmbed.startingCall(intent)) { + const createEvent = ct.state.join.create.read.find((evt) => evt.roomId === roomId)?.event; + const roomType = ObjectReader.string(createEvent?.content, 'type'); + if (roomType === RoomType.Call && CallEmbed.startingCall(intent)) { params.append('sendNotificationType', CallEmbed.dmCall(intent) ? 'ring' : 'notification'); } @@ -150,23 +147,21 @@ export class CallEmbed { } constructor( - mx: MatrixClient, - room: Room, + ct: ClientContext, + roomId: string, widget: Widget, container: HTMLElement, initialControlState?: CallControlState ) { - const iframe = CallEmbed.getIframe( - widget.getCompleteUrl({ currentUserId: mx.getSafeUserId() }) - ); + const iframe = CallEmbed.getIframe(widget.getCompleteUrl({ currentUserId: ct.session.userId })); container.append(iframe); - const callWidgetDriver: WidgetDriver = new CallWidgetDriver(mx, room.roomId); + const callWidgetDriver: WidgetDriver = new CallWidgetDriver(ct, roomId); const call: ClientWidgetApi = new ClientWidgetApi(widget, iframe, callWidgetDriver); - this.mx = mx; + this.ct = ct; this.call = call; - this.room = room; + this.roomId = roomId; this.iframe = iframe; this.container = container; @@ -178,7 +173,7 @@ export class CallEmbed { }; let initialMediaEvent = true; - this.disposables.push( + this.disposer.collect( this.listenAction(ElementWidgetActions.DeviceMute, (evt) => { if (initialMediaEvent) { initialMediaEvent = false; @@ -192,10 +187,6 @@ export class CallEmbed { this.start(); } - get roomId(): string { - return this.room.roomId; - } - get document(): Document | undefined { return this.iframe.contentDocument ?? this.iframe.contentWindow?.document; } @@ -229,180 +220,68 @@ export class CallEmbed { private start() { // Room widgets get locked to the room they were added in this.call.setViewedRoomId(this.roomId); - this.disposables.push( + this.disposer.collect( this.listenAction(ElementWidgetActions.JoinCall, this.onCallJoined.bind(this)) ); - // Populate the map of "read up to" events for this widget with the current event in every room. - // This is a bit inefficient, but should be okay. We do this for all rooms in case the widget - // requests timeline capabilities in other rooms down the road. It's just easier to manage here. - this.mx.getRooms().forEach((room) => { - // Timelines are most recent last - const events = room.getLiveTimeline()?.getEvents() || []; - const roomEvent = events[events.length - 1]; - if (!roomEvent) return; // force later code to think the room is fresh - this.readUpToMap[room.roomId] = roomEvent.getId()!; - }); + this.disposer.collect( + this.ct.toDeviceEvents.subscribe((toDeviceEvents) => { + this.handleToDeviceEvents(toDeviceEvents); + }), + this.ct.sync.observable.subscribe((syncValue) => { + const [, data] = syncValue ?? []; + if (!data) return; - // Attach listeners for feeding events - the underlying widget classes handle permissions for us - this.mx.on(ClientEvent.Event, this.onEvent.bind(this)); - this.mx.on(MatrixEventEvent.Decrypted, this.onEventDecrypted.bind(this)); - this.mx.on(RoomStateEvent.Events, this.onStateUpdate.bind(this)); - this.mx.on(ClientEvent.ToDeviceEvent, this.onToDeviceEvent.bind(this)); + const roomData = data.rooms?.join?.[this.roomId]; + const events = roomData?.timeline?.events; + + if (!events) return; + this.handleTimelineEvents(events); + }) + ); } - /** - * Stops the widget messaging for if it is started. Skips stopping if it is an active - * widget. - * @param opts - */ - public dispose(): void { - this.disposables.forEach((disposable) => { - disposable(); + private handleToDeviceEvents(toDeviceEvents: ProcessedToDeviceEvent[]): void { + toDeviceEvents.forEach((toDevice) => { + const encrypted = toDevice instanceof ToDeviceDecrypted || toDevice instanceof ToDeviceUTD; + this.call.feedToDevice(toDevice.event as ToDeviceEvent>, encrypted); }); - this.call.stop(); - this.container.removeChild(this.iframe); - this.control.dispose(); + } - this.mx.off(ClientEvent.Event, this.onEvent.bind(this)); - this.mx.off(MatrixEventEvent.Decrypted, this.onEventDecrypted.bind(this)); - this.mx.off(RoomStateEvent.Events, this.onStateUpdate.bind(this)); - this.mx.off(ClientEvent.ToDeviceEvent, this.onToDeviceEvent.bind(this)); + private handleTimelineEvents(events: ClientEventWithoutRoomId[]): void { + events.forEach(async (event) => { + if (this.ct.encryptedRooms.read.has(this.roomId)) { + const decryptedEvent = await this.ct.decryption.tryDecryptEvent(this.roomId, event); + this.feedEvent(decryptedEvent); + return; + } + this.feedEvent(event); + }); + } - // Clear internal state - this.readUpToMap = {}; - this.eventsToFeed = new WeakSet(); + private feedEvent(event: ClientEventWithoutRoomId): void { + if ('state_key' in event) { + this.call.feedStateUpdate({ + ...event, + room_id: this.roomId, + content: event.content as Record, + unsigned: event.unsigned as Record, + }); + return; + } + + this.call.feedEvent({ + ...event, + room_id: this.roomId, + content: event.content as Record, + unsigned: event.unsigned as Record, + }); } private onCallJoined(): void { this.joined = true; } - private onEvent(ev: MatrixEvent): void { - this.mx.decryptEventIfNeeded(ev); - this.feedEvent(ev); - } - - private onEventDecrypted(ev: MatrixEvent): void { - this.feedEvent(ev); - } - - private onStateUpdate(ev: MatrixEvent): void { - if (this.call === null) return; - const raw = ev.getEffectiveEvent(); - this.call.feedStateUpdate(raw as IRoomEvent).catch((e) => { - console.error('Error sending state update to widget: ', e); - }); - } - - private async onToDeviceEvent(ev: MatrixEvent): Promise { - await this.mx.decryptEventIfNeeded(ev); - if (ev.isDecryptionFailure()) return; - await this.call?.feedToDevice(ev.getEffectiveEvent() as IRoomEvent, ev.isEncrypted()); - } - - /** - * Determines whether the event has a relation to an unknown parent. - */ - private relatesToUnknown(ev: MatrixEvent): boolean { - // Replies to unknown events don't count - if (!ev.relationEventId || ev.replyEventId) return false; - const room = this.mx.getRoom(ev.getRoomId()); - return room === null || !room.findEventById(ev.relationEventId); - } - - /** - * Advances the "read up to" marker for a room to a certain event. No-ops if - * the event is before the marker. - * @returns Whether the "read up to" marker was advanced. - */ - private advanceReadUpToMarker(ev: MatrixEvent): boolean { - const evId = ev.getId(); - if (evId === undefined) return false; - const roomId = ev.getRoomId(); - if (roomId === undefined) return false; - const room = this.mx.getRoom(roomId); - if (room === null) return false; - - const upToEventId = this.readUpToMap[ev.getRoomId()!]; - if (!upToEventId) { - // There's no marker yet; start it at this event - this.readUpToMap[roomId] = evId; - return true; - } - - // Small optimization for exact match (skip the search) - if (upToEventId === evId) return false; - - // Timelines are most recent last, so reverse the order and limit ourselves to 100 events - // to avoid overusing the CPU. - const timeline = room.getLiveTimeline(); - const events = [...timeline.getEvents()].reverse().slice(0, 100); - function isRelevantTimelineEvent(timelineEvent: MatrixEvent): boolean { - return timelineEvent.getId() === upToEventId || timelineEvent.getId() === ev.getId(); - } - const possibleMarkerEv = events.find(isRelevantTimelineEvent); - if (possibleMarkerEv?.getId() === upToEventId) { - // The event must be somewhere before the "read up to" marker - return false; - } - if (possibleMarkerEv?.getId() === ev.getId()) { - // The event is after the marker; advance it - this.readUpToMap[roomId] = evId; - return true; - } - - // We can't say for sure whether the widget has seen the event; let's - // just assume that it has - return false; - } - - /** - * Determines whether the event comes from a room that we've been invited to - * (in which case we likely don't have the full timeline). - */ - private isFromInvite(ev: MatrixEvent): boolean { - const room = this.mx.getRoom(ev.getRoomId()); - return room?.getMyMembership() === KnownMembership.Invite; - } - - private feedEvent(ev: MatrixEvent): void { - if (this.call === null) return; - if ( - // If we had decided earlier to feed this event to the widget, but - // it just wasn't ready, give it another try - this.eventsToFeed.delete(ev) || - // Skip marker timeline check for events with relations to unknown parent because these - // events are not added to the timeline here and will be ignored otherwise: - // https://github.com/matrix-org/matrix-js-sdk/blob/d3dfcd924201d71b434af3d77343b5229b6ed75e/src/models/room.ts#L2207-L2213 - this.relatesToUnknown(ev) || - // Skip marker timeline check for rooms where membership is - // 'invite', otherwise the membership event from the invitation room - // will advance the marker and new state events will not be - // forwarded to the widget. - this.isFromInvite(ev) || - // Check whether this event would be before or after our "read up to" marker. If it's - // before, or we can't decide, then we assume the widget will have already seen the event. - // If the event is after, or we don't have a marker for the room, - // then the marker will advance and we'll send it through. - // This approach of "read up to" prevents widgets receiving decryption spam from startup or - // receiving ancient events from backfill and such. - this.advanceReadUpToMarker(ev) - ) { - // If the event is still being decrypted, remember that we want to - // feed it to the widget (even if not strictly in the order given by - // the timeline) and get back to it later - if (ev.isBeingDecrypted() || ev.isDecryptionFailure()) { - this.eventsToFeed.add(ev); - } else { - const raw = ev.getEffectiveEvent(); - this.call.feedEvent(raw as IRoomEvent).catch((e) => { - console.error('Error sending event to widget: ', e); - }); - } - } - } - public listenAction(type: string, callback: (event: CustomEvent) => void) { return this.listenEvent(`action:${type}`, callback); } @@ -413,4 +292,16 @@ export class CallEmbed { this.call.off(type, callback); }; } + + /** + * Stops the widget messaging for if it is started. Skips stopping if it is an active + * widget. + * @param opts + */ + public dispose(): void { + this.disposer.run(); + this.call.stop(); + this.container.removeChild(this.iframe); + this.control.dispose(); + } } diff --git a/src/app/plugins/call/CallWidgetDriver.ts b/src/app/plugins/call/CallWidgetDriver.ts index d21eaa7b..8938b493 100644 --- a/src/app/plugins/call/CallWidgetDriver.ts +++ b/src/app/plugins/call/CallWidgetDriver.ts @@ -11,34 +11,75 @@ import { OpenIDRequestState, SimpleObservable, IOpenIDUpdate, + IRoomAccountData, + IRtcTransportsResult, + IRtcLivekitGetTokenFromWidgetRequestData, + IRtcLivekitGetTokenFromWidgetResponseData, + IRtcLivekitDelegateDelayedLeaveFromWidgetRequestData, + IRtcLivekitDelegateDelayedLeaveFromWidgetResponseData, } from 'matrix-widget-api'; import { - EventType, - type IContent, - MatrixError, - type MatrixEvent, - Direction, - type SendDelayedEventResponse, - type StateEvents, - type TimelineEvents, - MatrixClient, -} from 'matrix-js-sdk'; + getMessages, + GetMessagesOK, + putRedact, + PutRedactOK, + putSend, + PutSendOK, + putStateEvent, + PutStateEventOK, +} from 'neurone/lib/api/matrix/client/v3/rooms'; +import { parseMatrixResp } from 'neurone/lib/matrix-request'; +import { putSendToDevice } from 'neurone/lib/api/matrix/client/v3/sendToDevice'; +import { postUpload, PostUploadOK } from 'neurone/lib/api/matrix/media/v3/upload'; +import { getConfig, GetConfigOK } from 'neurone/lib/api/matrix/client/v1/media'; +import { postSearch, PostSearchOK } from 'neurone/lib/api/matrix/client/v3/userDirectory'; +import { + postOpenidRequestToken, + PostOpenidRequestTokenOK, +} from 'neurone/lib/api/matrix/client/v3/user'; +import { + getRelations, + GetRelationsOK, + getRelationsRelType, + getRelationsRelTypeEventType, +} from 'neurone/lib/api/matrix/client/v1/rooms'; +import { QueryRange } from 'neurone/lib/storage/collection'; +import { MatrixEndpointStruct } from 'neurone/lib/api/MatrixEndpoint'; +import { ClientEventWithoutRoomId } from 'neurone/lib/api/types'; import { getCallCapabilities } from './utils'; -import { downloadMedia, mxcUrlToHttp } from '../../utils/matrix'; +import { downloadMedia, getMxIdServer } from '../../utils/matrix'; +import { ClientContext } from '../../../client/context'; +import { MessageEvent } from '../../../types/matrix/room'; +import { mxcToHttp } from '../../../client/mxcToHttp'; +import { getAuthUploadRequest } from '../../../client/request'; +import { getJoinRoomMemberId, getJoinRoomStateId } from '../../../client/storage/collections'; +import { MAX_UNICODE } from '../../../client/storage'; +import { rContent } from '../../../client/reader'; +import { + unstablePostDelayedEvents, + unstablePutDelayedEvent, + UnstablePutDelayedEventOK, +} from '../msc4140-delayed-events'; +import { fulfilledPromiseSettledResult } from '../../utils/common'; +import { ensureEncryptionSession } from '../../hooks/useEnsureRoomEncryptionSession'; +import { unstablePutSendSticky } from '../msc4354-sticky-events'; +import { unstableGetTransports, UnstableGetTransportsOK } from '../msc4143-matrixrtc'; +import { + unstablePostLivekitDelegateDelayedLeave, + unstablePostLivekitGetToken, + UnstablePostLivekitGetTokenOK, +} from '../msc4195-matrixrtc-livekit'; +import { autoDiscovery } from '../../cs-api'; export class CallWidgetDriver extends WidgetDriver { private allowedCapabilities: Set; - private readonly mx: MatrixClient; - - public constructor(mx: MatrixClient, private inRoomId: string) { + public constructor(private ct: ClientContext, private inRoomId: string) { super(); - this.mx = mx; - const deviceId = mx.getDeviceId(); - if (!deviceId) throw new Error('Failed to initialize CallWidgetDriver! Device ID not found.'); + const { deviceId, userId } = ct.session; - this.allowedCapabilities = getCallCapabilities(inRoomId, mx.getSafeUserId(), deviceId); + this.allowedCapabilities = getCallCapabilities(inRoomId, userId, deviceId); } public async validateCapabilities(requested: Set): Promise> { @@ -48,95 +89,217 @@ export class CallWidgetDriver extends WidgetDriver { public async sendEvent( eventType: string, - content: IContent, + content: object, stateKey: string | null = null, targetRoomId: string | null = null ): Promise { const roomId = targetRoomId || this.inRoomId; - let r: { event_id: string } | null; if (typeof stateKey === 'string') { - r = await this.mx.sendStateEvent( - roomId, - eventType as keyof StateEvents, - content as StateEvents[keyof StateEvents], - stateKey - ); - } else if (eventType === EventType.RoomRedaction) { - // special case: extract the `redacts` property and call redact - r = await this.mx.redactEvent(roomId, content.redacts); - } else { - r = await this.mx.sendEvent( - roomId, - eventType as keyof TimelineEvents, - content as TimelineEvents[keyof TimelineEvents] + const resp = await this.ct.requestAuthJSON( + putStateEvent({ pathParams: { roomId, eventType, stateKey }, body: content }) ); + const [err, result] = await parseMatrixResp(resp); + if (err) throw err; + + return { roomId, eventId: result.event_id }; } - return { roomId, eventId: r.event_id }; + if ( + eventType === MessageEvent.RoomRedaction && + 'redacts' in content && + typeof content.redacts === 'string' + ) { + // special case: extract the `redacts` property and call redact + const resp = await this.ct.requestAuthJSON( + putRedact({ pathParams: { roomId, eventId: content.redacts, txnId: this.ct.getTxnId() } }) + ); + const [err, result] = await parseMatrixResp(resp); + if (err) throw err; + + return { roomId, eventId: result.event_id }; + } + + let encContent: object | undefined; + if (this.ct.encryptedRooms.read.has(roomId)) { + await ensureEncryptionSession(this.ct, roomId); + + const encStr = await this.ct.machine.roomEncryption.encryptMessageEvent( + roomId, + eventType, + content + ); + encContent = JSON.parse(encStr); + } + + const resp = await this.ct.requestAuthJSON( + putSend({ + pathParams: { + roomId, + eventType: encContent ? MessageEvent.RoomMessageEncrypted : eventType, + txnId: this.ct.getTxnId(), + }, + body: encContent ?? content, + }) + ); + const [err, result] = await parseMatrixResp(resp); + if (err) throw err; + + return { roomId, eventId: result.event_id }; + } + + public async sendStickyEvent( + stickyDurationMs: number, + eventType: string, + content: object, + targetRoomId?: string | null + ): Promise { + const roomId = targetRoomId || this.inRoomId; + + let encContent: object | undefined; + if (this.ct.encryptedRooms.read.has(roomId)) { + await ensureEncryptionSession(this.ct, roomId); + + const encStr = await this.ct.machine.roomEncryption.encryptMessageEvent( + roomId, + eventType, + content + ); + encContent = JSON.parse(encStr); + } + + const resp = await this.ct.requestAuthJSON( + unstablePutSendSticky({ + pathParams: { + roomId, + eventType: encContent ? MessageEvent.RoomMessageEncrypted : eventType, + txnId: this.ct.getTxnId(), + }, + searchParams: { + 'org.matrix.msc4354.sticky_duration_ms': stickyDurationMs, + }, + body: encContent ?? content, + }) + ); + const [err, result] = await parseMatrixResp(resp); + if (err) throw err; + + return { roomId, eventId: result.event_id }; } public async sendDelayedEvent( - delay: number | null, - parentDelayId: string | null, + delay: number, eventType: string, - content: IContent, + content: object, stateKey: string | null = null, targetRoomId: string | null = null ): Promise { const roomId = targetRoomId || this.inRoomId; - let delayOpts; - if (delay !== null) { - delayOpts = { - delay, - ...(parentDelayId !== null && { parent_delay_id: parentDelayId }), - }; - } else if (parentDelayId !== null) { - delayOpts = { - parent_delay_id: parentDelayId, - }; - } else { - throw new Error('Must provide at least one of delay or parentDelayId'); - } + let encContent: object | undefined; + if (typeof stateKey !== 'string' && this.ct.encryptedRooms.read.has(roomId)) { + await ensureEncryptionSession(this.ct, roomId); - let r: SendDelayedEventResponse | null; - if (stateKey !== null) { - // state event - r = await this.mx._unstable_sendDelayedStateEvent( + const encStr = await this.ct.machine.roomEncryption.encryptMessageEvent( roomId, - delayOpts, - eventType as keyof StateEvents, - content as StateEvents[keyof StateEvents], - stateKey - ); - } else { - // message event - r = await this.mx._unstable_sendDelayedEvent( - roomId, - delayOpts, - null, - eventType as keyof TimelineEvents, - content as TimelineEvents[keyof TimelineEvents] + eventType, + content ); + encContent = JSON.parse(encStr); + } + const resp = await this.ct.requestAuthJSON( + unstablePutDelayedEvent({ + pathParams: { + roomId, + eventType: encContent ? MessageEvent.RoomMessageEncrypted : eventType, + txnId: this.ct.getTxnId(), + }, + body: { + delay_ms: delay, + state_key: stateKey ?? undefined, + content: encContent ?? content, + }, + }) + ); + + const [err, result] = await parseMatrixResp(resp); + + if (err) { + throw err; } return { roomId, - delayId: r.delay_id, + delayId: result.delay_id, + }; + } + + public async sendDelayedStickyEvent( + delay: number, + stickyDurationMs: number, + eventType: string, + content: object, + targetRoomId?: string | null + ): Promise { + const roomId = targetRoomId || this.inRoomId; + + let encContent: object | undefined; + if (this.ct.encryptedRooms.read.has(roomId)) { + await ensureEncryptionSession(this.ct, roomId); + + const encStr = await this.ct.machine.roomEncryption.encryptMessageEvent( + roomId, + eventType, + content + ); + encContent = JSON.parse(encStr); + } + const resp = await this.ct.requestAuthJSON( + unstablePutDelayedEvent({ + pathParams: { + roomId, + eventType: encContent ? MessageEvent.RoomMessageEncrypted : eventType, + txnId: this.ct.getTxnId(), + }, + body: { + delay_ms: delay, + 'org.matrix.msc4354.sticky_duration_ms': stickyDurationMs, + content: encContent ?? content, + }, + }) + ); + + const [err, result] = await parseMatrixResp(resp); + + if (err) { + throw err; + } + + return { + roomId, + delayId: result.delay_id, }; } public async cancelScheduledDelayedEvent(delayId: string): Promise { - await this.mx._unstable_cancelScheduledDelayedEvent(delayId); + const resp = await this.ct.requestAuthJSON( + unstablePostDelayedEvents({ pathParams: { delayId, action: 'cancel' } }) + ); + await parseMatrixResp(resp); } public async restartScheduledDelayedEvent(delayId: string): Promise { - await this.mx._unstable_restartScheduledDelayedEvent(delayId); + const resp = await this.ct.requestAuthJSON( + unstablePostDelayedEvents({ pathParams: { delayId, action: 'restart' } }) + ); + await parseMatrixResp(resp); } public async sendScheduledDelayedEvent(delayId: string): Promise { - await this.mx._unstable_sendScheduledDelayedEvent(delayId); + const resp = await this.ct.requestAuthJSON( + unstablePostDelayedEvents({ pathParams: { delayId, action: 'send' } }) + ); + await parseMatrixResp(resp); } public async sendToDevice( @@ -144,48 +307,89 @@ export class CallWidgetDriver extends WidgetDriver { encrypted: boolean, contentMap: { [userId: string]: { [deviceId: string]: object } } ): Promise { + let messages = contentMap; + if (encrypted) { - const crypto = this.mx.getCrypto(); - if (!crypto) throw new Error('E2EE not enabled'); + const users = Object.keys(contentMap); + await this.ct.machine.outgoingRequests.claimKeys(users); - // attempt to re-batch these up into a single request - const invertedContentMap: { [content: string]: { userId: string; deviceId: string }[] } = {}; + const encryptedContentMap: { [userId: string]: { [deviceId: string]: object } } = {}; - // eslint-disable-next-line no-restricted-syntax - for (const userId of Object.keys(contentMap)) { - const userContentMap = contentMap[userId]; - // eslint-disable-next-line no-restricted-syntax - for (const deviceId of Object.keys(userContentMap)) { - const content = userContentMap[deviceId]; - const stringifiedContent = JSON.stringify(content); - invertedContentMap[stringifiedContent] = invertedContentMap[stringifiedContent] || []; - invertedContentMap[stringifiedContent].push({ userId, deviceId }); - } - } + await Promise.allSettled( + Object.entries(contentMap).map(async ([userId, userContentMap]) => { + encryptedContentMap[userId] = {}; - await Promise.all( - Object.entries(invertedContentMap).map(async ([stringifiedContent, recipients]) => { - const batch = await crypto.encryptToDeviceMessages( - eventType, - recipients, - JSON.parse(stringifiedContent) + await Promise.all( + Object.entries(userContentMap).map(async ([deviceId, payload]) => { + const userDevice = await this.ct.machine.userTracker.getDevice(userId, deviceId); + + if (!userDevice) { + throw new Error(`Device not found: ${userId} / ${deviceId}`); + } + + try { + const encryptedPayload = userDevice.encryptToDeviceEvent(eventType, payload); + + if (encryptedPayload == null || typeof encryptedPayload !== 'object') { + throw new Error('Encryption returned an invalid payload'); + } + + encryptedContentMap[userId][deviceId] = encryptedPayload; + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + + throw new Error( + `Failed to encrypt to-device event for ${userId} / ${deviceId}: ${reason}`, + { cause: error } + ); + } + }) ); - - await this.mx.queueToDevice(batch); }) ); - } else { - await this.mx.queueToDevice({ - eventType, - batch: Object.entries(contentMap).flatMap(([userId, userContentMap]) => - Object.entries(userContentMap).map(([deviceId, content]) => ({ - userId, - deviceId, - payload: content, - })) - ), - }); + + messages = encryptedContentMap; } + + const resp = await this.ct.requestAuthJSON( + putSendToDevice({ + pathParams: { eventType, txnId: this.ct.getTxnId() }, + body: { messages }, + }) + ); + + const [err] = await parseMatrixResp(resp); + + if (err) { + throw err; + } + } + + public async readRoomAccountData( + eventType: string, + roomIds?: string[] | null + ): Promise { + const dataObjects = await this.ct.storage.joinAccountData.read.queryIndex('type', eventType); + const events = dataObjects + .filter((obj) => (Array.isArray(roomIds) ? roomIds.includes(obj.roomId) : true)) + .map((obj) => ({ + ...obj.event, + room_id: obj.roomId, + content: obj.event.content as Record, + })); + + return events; + } + + public async readStickyEvents(roomId: string): Promise { + const eventObjs = await this.ct.storage.joinRoomSticky.read.queryIndex('roomId', roomId); + + return eventObjs.map(({ event }) => ({ + room_id: roomId, + ...event, + content: event.content as Record, + unsigned: event.unsigned as Record, + })); } public async readRoomTimeline( @@ -193,40 +397,54 @@ export class CallWidgetDriver extends WidgetDriver { eventType: string, msgtype: string | undefined, stateKey: string | undefined, - limit: number, - since: string | undefined + limit: number ): Promise { const safeLimit = - limit > 0 ? Math.min(limit, Number.MAX_SAFE_INTEGER) : Number.MAX_SAFE_INTEGER; // relatively arbitrary + limit > 0 ? Math.min(limit, Number.MAX_SAFE_INTEGER) : Number.MAX_SAFE_INTEGER; - const room = this.mx.getRoom(roomId); - if (room === null) return []; - const results: MatrixEvent[] = []; - const events = room.getLiveTimeline().getEvents(); + const msgResp = await this.ct.requestAuthJSON( + getMessages({ + pathParams: { roomId }, + searchParams: { dir: 'b', limit: safeLimit }, + }) + ); - for (let i = events.length - 1; i >= 0; i -= 1) { + const [err, result] = await parseMatrixResp(msgResp); + if (err) throw err; + + let events: ClientEventWithoutRoomId[] = result.chunk; + + if (this.ct.encryptedRooms.read.has(roomId)) { + const pending = events.map((evt) => this.ct.decryption.tryDecryptEvent(roomId, evt)); + + events = fulfilledPromiseSettledResult(await Promise.allSettled(pending)); + } + + const resultEvents: ClientEventWithoutRoomId[] = []; + + for (let i = 0; i < events.length; i += 1) { const ev = events[i]; - if (results.length >= safeLimit) break; - if (since !== undefined && ev.getId() === since) break; + if (resultEvents.length >= safeLimit) break; + + const eventStateKey = 'state_key' in ev ? ev.state_key : undefined; if ( - ev.getType() === eventType && - !ev.isState() && - (eventType !== EventType.RoomMessage || !msgtype || msgtype === ev.getContent().msgtype) && - (ev.getStateKey() === undefined || stateKey === undefined || ev.getStateKey() === stateKey) + ev.type === eventType && + (eventStateKey === undefined || stateKey === undefined || eventStateKey === stateKey) && + (eventType !== MessageEvent.RoomMessage || + !msgtype || + msgtype === rContent.msgtype(ev.content)) ) { - results.push(ev); + resultEvents.push(ev); } } - return results.map((e) => e.getEffectiveEvent() as IRoomEvent); - } - - public async askOpenID(observer: SimpleObservable): Promise { - return observer.update({ - state: OpenIDRequestState.Allowed, - token: await this.mx.getOpenIdToken(), - }); + return resultEvents.map((evt) => ({ + room_id: roomId, + ...evt, + content: evt.content as Record, + unsigned: evt.unsigned as Record, + })); } public async readRoomState( @@ -234,58 +452,134 @@ export class CallWidgetDriver extends WidgetDriver { eventType: string, stateKey: string | undefined ): Promise { - const room = this.mx.getRoom(roomId); - if (room === null) return []; - const state = room.getLiveTimeline().getState(Direction.Forward); - if (state === undefined) return []; + if (stateKey === undefined) { + const stateObjs = + eventType === 'm.room.member' + ? await this.ct.storage.joinRoomMember.read.queryIndex('roomId', roomId) + : await this.ct.storage.joinRoomState.read.query( + new QueryRange( + getJoinRoomStateId(roomId, eventType, ''), + getJoinRoomStateId(roomId, eventType, MAX_UNICODE) + ) + ); - if (stateKey === undefined) - return state.getStateEvents(eventType).map((e) => e.getEffectiveEvent() as IRoomEvent); - const event = state.getStateEvents(eventType, stateKey); - return event === null ? [] : [event.getEffectiveEvent() as IRoomEvent]; + return stateObjs.map((stateObj) => ({ + ...stateObj.event, + room_id: roomId, + content: stateObj.event.content as Record, + unsigned: stateObj.event.unsigned as Record, + })); + } + + const stateObj = + eventType === 'm.room.member' + ? await this.ct.storage.joinRoomMember.read.get(getJoinRoomMemberId(roomId, stateKey)) + : await this.ct.storage.joinRoomState.read.get( + getJoinRoomStateId(roomId, eventType, stateKey ?? '') + ); + if (!stateObj) return []; + const { event: stateEvent } = stateObj; + + return [ + { + ...stateEvent, + room_id: roomId, + content: stateEvent.content as Record, + unsigned: stateEvent.unsigned as Record, + }, + ]; } public async readEventRelations( eventId: string, roomId?: string, - relationType?: string, + relType?: string, eventType?: string, from?: string, to?: string, limit?: number, - direction?: 'f' | 'b' + dir?: 'f' | 'b' ): Promise { - const dir = direction as Direction; - const targetRoomId = roomId ?? this.inRoomId ?? undefined; + const targetRoomId = roomId ?? this.inRoomId; if (typeof targetRoomId !== 'string') { throw new Error('Error while reading the current room'); } - const { events, nextBatch, prevBatch } = await this.mx.relations( - targetRoomId, - eventId, - relationType ?? null, - eventType ?? null, - { from, to, limit, dir } - ); + let endpoint: MatrixEndpointStruct; + if (relType && eventType) { + endpoint = getRelationsRelTypeEventType({ + pathParams: { roomId: targetRoomId, eventId, relType, eventType }, + searchParams: { dir, from, to, limit }, + }); + } else if (relType) { + endpoint = getRelationsRelType({ + pathParams: { roomId: targetRoomId, eventId, relType }, + searchParams: { dir, from, to, limit }, + }); + } else { + endpoint = getRelations({ + pathParams: { roomId: targetRoomId, eventId }, + searchParams: { dir, from, to, limit }, + }); + } - return { - chunk: events.map((e) => e.getEffectiveEvent() as IRoomEvent), - nextBatch: nextBatch ?? undefined, - prevBatch: prevBatch ?? undefined, - }; + const resp = await this.ct.requestAuthJSON(endpoint); + + const [err, result] = await parseMatrixResp(resp); + + if (err) { + throw err; + } + + if (this.ct.encryptedRooms.read.has(targetRoomId)) { + const pending = result.chunk.map((evt) => + this.ct.decryption.tryDecryptEvent(targetRoomId, evt) + ); + const clientEvents = fulfilledPromiseSettledResult(await Promise.allSettled(pending)); + return { + chunk: clientEvents.map((evt) => ({ + room_id: targetRoomId, + ...evt, + content: evt.content as Record, + unsigned: evt.unsigned as Record, + })), + prevBatch: result.prev_batch, + nextBatch: result.next_batch, + }; + } + return result as IReadEventRelationsResult; + } + + public async askOpenID(observer: SimpleObservable): Promise { + const resp = await this.ct.requestAuthJSON( + postOpenidRequestToken({ + pathParams: { + userId: this.ct.session.userId, + }, + }) + ); + const [err, result] = await parseMatrixResp(resp); + if (err) return; + + observer.update({ state: OpenIDRequestState.Allowed, token: result }); } public async searchUserDirectory( searchTerm: string, limit?: number ): Promise { - const { limited, results } = await this.mx.searchUserDirectory({ term: searchTerm, limit }); + const resp = await this.ct.requestAuthJSON( + postSearch({ body: { search_term: searchTerm, limit } }) + ); + const [err, result] = await parseMatrixResp(resp); + if (err) { + throw err; + } return { - limited, - results: results.map((r) => ({ + limited: result.limited, + results: result.results.map((r) => ({ userId: r.user_id, displayName: r.display_name, avatarUrl: r.avatar_url, @@ -294,32 +588,95 @@ export class CallWidgetDriver extends WidgetDriver { } public async getMediaConfig(): Promise { - return this.mx.getMediaConfig(); + const resp = await this.ct.requestAuthJSON(getConfig({})); + const [err, result] = await parseMatrixResp(resp); + if (err) { + throw err; + } + return result; + } + + public async getRtcTransports(): Promise { + const resp = await this.ct.requestAuthJSON(unstableGetTransports({})); + const [err, result] = await parseMatrixResp(resp); + if (result) { + return { + rtc_transports: result.rtc_transports ?? result.transports, + }; + } + + const userId = getMxIdServer(this.ct.session.userId); + if (!userId) throw err; + + const [, info] = await autoDiscovery(fetch, userId); + const rtcInfo = info?.['org.matrix.msc4143.rtc_foci']; + if (rtcInfo) + return { + rtc_transports: rtcInfo, + }; + + throw err; } public async uploadFile(file: XMLHttpRequestBodyInit): Promise<{ contentUri: string }> { - const uploadResult = await this.mx.uploadContent(file); + if (!(file instanceof Blob) || !(file instanceof FormData)) { + throw new Error('Failed to upload file. Unsupported file type'); + } - return { contentUri: uploadResult.content_uri }; + const request = getAuthUploadRequest( + this.ct.session.baseUrl, + this.ct.session.accessToken + ); + const [err, result] = await request(postUpload({ body: file })); + if (err) { + throw err; + } + return { contentUri: result.content_uri }; } public async downloadFile(contentUri: string): Promise<{ file: XMLHttpRequestBodyInit }> { - const httpUrl = mxcUrlToHttp(this.mx, contentUri, true); - if (!httpUrl) { - throw new Error('Call widget failed to download file! No http url!'); - } + const httpUrl = mxcToHttp(this.ct.session.baseUrl, contentUri, true).url(); const blob = await downloadMedia(httpUrl); return { file: blob }; } + public async getRtcLivekitToken( + data: IRtcLivekitGetTokenFromWidgetRequestData + ): Promise { + const resp = await this.ct.requestAuthJSON( + unstablePostLivekitGetToken({ + body: data, + }) + ); + const [err, result] = await parseMatrixResp(resp); + if (err) { + throw err; + } + return result; + } + + public async delegateRtcLivekitDelayedLeave( + data: IRtcLivekitDelegateDelayedLeaveFromWidgetRequestData + ): Promise { + const resp = await this.ct.requestAuthJSON( + unstablePostLivekitDelegateDelayedLeave({ + body: data, + }) + ); + const [err] = await parseMatrixResp(resp); + if (err) { + throw err; + } + + return {}; + } + public getKnownRooms(): string[] { - return this.mx.getVisibleRooms().map((r) => r.roomId); + return this.ct.rooms.join.read; } // eslint-disable-next-line class-methods-use-this - public processError(error: unknown): IWidgetApiErrorResponseDataDetails | undefined { - return error instanceof MatrixError - ? { matrix_api_error: error.asWidgetApiErrorData() } - : undefined; + public processError(): IWidgetApiErrorResponseDataDetails | undefined { + return undefined; } } diff --git a/src/app/plugins/call/hooks.ts b/src/app/plugins/call/hooks.ts index f5b3ff93..4d8a6c5c 100644 --- a/src/app/plugins/call/hooks.ts +++ b/src/app/plugins/call/hooks.ts @@ -3,9 +3,10 @@ import { IWidgetApiAcknowledgeResponseData, IWidgetApiRequestData, } from 'matrix-widget-api'; -import { useCallback, useEffect, useState } from 'react'; -import { CallControl, CallControlEvent } from './CallControl'; +import { useCallback, useEffect } from 'react'; +import { CallControl } from './CallControl'; import { CallControlState } from './CallControlState'; +import { useObservableValue } from '../../hooks/useObservableValue'; export const useClientWidgetApiEvent = ( api: ClientWidgetApi | undefined, @@ -33,17 +34,7 @@ export const useSendClientWidgetApiAction = (api: ClientWidgetApi) => { }; export const useCallControlState = (control: CallControl): CallControlState => { - const [state, setState] = useState(control.getState()); - - useEffect(() => { - const handleUpdate = () => { - setState(control.getState()); - }; - control.on(CallControlEvent.StateUpdate, handleUpdate); - return () => { - control.off(CallControlEvent.StateUpdate, handleUpdate); - }; - }, [control]); + const state = useObservableValue(control.state); return state; }; diff --git a/src/app/plugins/call/utils.ts b/src/app/plugins/call/utils.ts index 58d6d967..b0fc5013 100644 --- a/src/app/plugins/call/utils.ts +++ b/src/app/plugins/call/utils.ts @@ -4,7 +4,7 @@ import { MatrixCapabilities, WidgetEventCapability, } from 'matrix-widget-api'; -import { EventType } from 'matrix-js-sdk'; +import { CallEventType, MessageEvent, StateEventType } from '../../../types/matrix/room'; export function getCallCapabilities( roomId: string, @@ -20,20 +20,23 @@ export function getCallCapabilities( capabilities.add(MatrixCapabilities.MSC3846TurnServers); capabilities.add(MatrixCapabilities.MSC4157SendDelayedEvent); capabilities.add(MatrixCapabilities.MSC4157UpdateDelayedEvent); + capabilities.add(MatrixCapabilities.MSC4407SendStickyEvent); + capabilities.add(MatrixCapabilities.MSC4407ReceiveStickyEvent); + capabilities.add(MatrixCapabilities.MSC4515RtcTransports); capabilities.add(`org.matrix.msc2762.timeline:${roomId}`); capabilities.add(`org.matrix.msc2762.state:${roomId}`); capabilities.add( - WidgetEventCapability.forStateEvent(EventDirection.Receive, EventType.RoomMember).raw + WidgetEventCapability.forStateEvent(EventDirection.Receive, StateEventType.RoomMember).raw ); capabilities.add( WidgetEventCapability.forStateEvent(EventDirection.Receive, 'org.matrix.msc3401.call').raw ); capabilities.add( - WidgetEventCapability.forStateEvent(EventDirection.Receive, EventType.RoomEncryption).raw + WidgetEventCapability.forStateEvent(EventDirection.Receive, StateEventType.RoomEncryption).raw ); capabilities.add( - WidgetEventCapability.forStateEvent(EventDirection.Receive, EventType.RoomName).raw + WidgetEventCapability.forStateEvent(EventDirection.Receive, StateEventType.RoomName).raw ); capabilities.add( @@ -77,34 +80,36 @@ export function getCallCapabilities( .raw ); capabilities.add( - WidgetEventCapability.forStateEvent(EventDirection.Receive, EventType.RoomCreate).raw + WidgetEventCapability.forStateEvent(EventDirection.Receive, StateEventType.RoomCreate).raw ); [ 'io.element.call.encryption_keys', 'org.matrix.rageshake_request', - EventType.Reaction, - EventType.RoomRedaction, + MessageEvent.Reaction, + MessageEvent.RoomRedaction, 'io.element.call.reaction', 'org.matrix.msc4075.rtc.notification', 'org.matrix.msc4310.rtc.decline', + 'org.matrix.msc4143.rtc.member', + 'org.matrix.msc4075.call.notify', ].forEach((type) => { capabilities.add(WidgetEventCapability.forRoomEvent(EventDirection.Send, type).raw); capabilities.add(WidgetEventCapability.forRoomEvent(EventDirection.Receive, type).raw); }); [ - EventType.CallInvite, - EventType.CallCandidates, - EventType.CallAnswer, - EventType.CallHangup, - EventType.CallReject, - EventType.CallSelectAnswer, - EventType.CallNegotiate, - EventType.CallSDPStreamMetadataChanged, - EventType.CallSDPStreamMetadataChangedPrefix, - EventType.CallReplaces, - EventType.CallEncryptionKeysPrefix, + CallEventType.CallInvite, + CallEventType.CallCandidates, + CallEventType.CallAnswer, + CallEventType.CallHangup, + CallEventType.CallReject, + CallEventType.CallSelectAnswer, + CallEventType.CallNegotiate, + CallEventType.CallSDPStreamMetadataChanged, + CallEventType.CallSDPStreamMetadataChangedPrefix, + CallEventType.CallReplaces, + CallEventType.CallEncryptionKeysPrefix, ].forEach((type) => { capabilities.add(WidgetEventCapability.forToDeviceEvent(EventDirection.Send, type).raw); capabilities.add(WidgetEventCapability.forToDeviceEvent(EventDirection.Receive, type).raw); diff --git a/src/app/plugins/msc4140-delayed-events.ts b/src/app/plugins/msc4140-delayed-events.ts new file mode 100644 index 00000000..04e4681d --- /dev/null +++ b/src/app/plugins/msc4140-delayed-events.ts @@ -0,0 +1,31 @@ +import { MatrixEndpoint } from 'neurone/lib/api/MatrixEndpoint'; + +export type UnstablePutDelayedEventOK = { delay_id: string }; + +export const unstablePutDelayedEvent: MatrixEndpoint<{ + pathParams: { roomId: string; eventType: string; txnId: string }; + body: + | { delay_ms: number; state_key?: string; content: object } + | { delay_ms: number; 'org.matrix.msc4354.sticky_duration_ms': number; content: object }; +}> = ({ pathParams, body }) => ({ + method: 'PUT', + path: `/_matrix/client/unstable/org.matrix.msc4140/rooms/${encodeURIComponent( + pathParams.roomId + )}/delayed_event/${encodeURIComponent(pathParams.eventType)}/${encodeURIComponent( + pathParams.txnId + )}`, + requireAuthentication: true, + rateLimited: true, + body: JSON.stringify(body), +}); + +export const unstablePostDelayedEvents: MatrixEndpoint<{ + pathParams: { delayId: string; action: 'send' | 'cancel' | 'restart' }; +}> = ({ pathParams }) => ({ + method: 'POST', + path: `/_matrix/client/unstable/org.matrix.msc4140/delayed_events/${encodeURIComponent( + pathParams.delayId + )}/${encodeURIComponent(pathParams.action)}`, + requireAuthentication: true, + rateLimited: true, +}); diff --git a/src/app/plugins/msc4143-matrixrtc.ts b/src/app/plugins/msc4143-matrixrtc.ts new file mode 100644 index 00000000..af14701a --- /dev/null +++ b/src/app/plugins/msc4143-matrixrtc.ts @@ -0,0 +1,21 @@ +import { MatrixEndpoint } from 'neurone/lib/api/MatrixEndpoint'; + +export type TransportInfo = { + type: string; + [key: string]: unknown; +}; +export type UnstableGetTransportsOK = { + transports: TransportInfo[]; + rtc_transports: TransportInfo[]; +}; + +export const unstableGetTransports: MatrixEndpoint = () => { + const path = `/_matrix/client/unstable/org.matrix.msc4143/rtc/transports`; + + return { + method: 'GET', + path, + requireAuthentication: true, + rateLimited: true, + }; +}; diff --git a/src/app/plugins/msc4195-matrixrtc-livekit.ts b/src/app/plugins/msc4195-matrixrtc-livekit.ts new file mode 100644 index 00000000..47f870a0 --- /dev/null +++ b/src/app/plugins/msc4195-matrixrtc-livekit.ts @@ -0,0 +1,45 @@ +import { MatrixEndpoint } from 'neurone/lib/api/MatrixEndpoint'; + +export type UnstablePostLivekitGetTokenOK = { + jwt: string; +}; + +export const unstablePostLivekitGetToken: MatrixEndpoint<{ + body: { + server_name?: string; + url: string; + room_id: string; + slot_id: string; + member_id: string; + }; +}> = ({ body }) => { + const path = `/_matrix/client/unstable/io.element.msc4195/rtc/livekit/get_token`; + + return { + method: 'POST', + path, + requireAuthentication: true, + rateLimited: true, + body: JSON.stringify(body), + }; +}; + +export const unstablePostLivekitDelegateDelayedLeave: MatrixEndpoint<{ + body: { + url: string; + room_id: string; + slot_id: string; + member_id: string; + delay_id: string; + }; +}> = ({ body }) => { + const path = `/_matrix/client/unstable/io.element.msc4195/rtc/livekit/delegate_delayed_leave`; + + return { + method: 'POST', + path, + requireAuthentication: true, + rateLimited: true, + body: JSON.stringify(body), + }; +}; diff --git a/src/app/plugins/msc4354-sticky-events.ts b/src/app/plugins/msc4354-sticky-events.ts new file mode 100644 index 00000000..e8ef0da0 --- /dev/null +++ b/src/app/plugins/msc4354-sticky-events.ts @@ -0,0 +1,22 @@ +import { PutSendBody, PutSendPathParams } from 'neurone/lib/api/matrix/client/v3/rooms'; +import { MatrixEndpoint } from 'neurone/lib/api/MatrixEndpoint'; +import { withSearchParams } from 'neurone/lib/api/utils'; + +export const unstablePutSendSticky: MatrixEndpoint<{ + pathParams: PutSendPathParams; + searchParams: { + 'org.matrix.msc4354.sticky_duration_ms': number; + }; + body: PutSendBody; +}> = ({ pathParams, searchParams, body }) => { + const path = `/_matrix/client/v3/rooms/${encodeURIComponent( + pathParams.roomId + )}/send/${encodeURIComponent(pathParams.eventType)}/${encodeURIComponent(pathParams.txnId)}`; + + return { + method: 'PUT', + path: withSearchParams(path, searchParams), + requireAuthentication: true, + body: JSON.stringify(body), + }; +}; diff --git a/src/app/state/upload.ts b/src/app/state/upload.ts index 21954c33..8fc1f3d8 100644 --- a/src/app/state/upload.ts +++ b/src/app/state/upload.ts @@ -121,7 +121,10 @@ export const useBindUploadAtom = (uploadAtom: UploadAtom, hideFilename?: boolean request( postUpload({ searchParams: { - filename: hideFilename ? undefined : file.name, + filename: + !hideFilename && 'name' in file && typeof file.name === 'string' + ? file.name + : undefined, }, body: file, }), diff --git a/src/app/utils/room.ts b/src/app/utils/room.ts index 57af4667..72dcce08 100644 --- a/src/app/utils/room.ts +++ b/src/app/utils/room.ts @@ -30,6 +30,8 @@ import { ClientContext } from '../../client/context'; import { rContent, rInReplyTo, rRelatesTo } from '../../client/reader'; import { replaced } from '../../client/room/relations'; +export const isCallRoom = (roomType?: string) => roomType === 'org.matrix.msc3417.call'; + export const getStateEvent = ( room: Room, eventType: StateEvent, diff --git a/src/client/SyncResponseManager.ts b/src/client/SyncResponseManager.ts index 60654454..3ec57aa8 100644 --- a/src/client/SyncResponseManager.ts +++ b/src/client/SyncResponseManager.ts @@ -2,7 +2,11 @@ import { GetSyncOK, JoinRoom } from 'neurone/lib/api/matrix/client/v3/sync'; import { ObservableValue } from 'neurone/lib/utils/observable'; import { EncryptionSyncChanges } from 'neurone/lib/crypto/machine'; import { Disposable, Disposer } from 'neurone/lib/utils/disposable'; -import { ToDeviceDecrypted, ToDevicePlainText } from 'neurone/lib/crypto/machine/events'; +import { + ProcessedToDeviceEvent, + ToDeviceDecrypted, + ToDevicePlainText, +} from 'neurone/lib/crypto/machine/events'; import { roomState } from 'neurone/lib/utils/sync/v3/join'; import { ObjectReader } from 'neurone/lib/utils'; import { InboundVerification, OutboundVerification } from 'neurone/lib/crypto/machine/verification'; @@ -17,6 +21,7 @@ export type SyncResponseManagerCallbacks = { onWriteComplete: () => void; onSyncToken: (token: string) => void; onVerificationRequest: (verificationFlow: OutboundVerification | InboundVerification) => void; + onToDeviceEvents: (toDeviceEvents: ProcessedToDeviceEvent[]) => void; }; export class SyncResponseManager implements Disposable { @@ -82,6 +87,8 @@ export class SyncResponseManager implements Disposable { this.callbacks.onVerificationRequest(v) ); }); + + this.callbacks.onToDeviceEvents(proccessedToDeviceEvents); // TODO: room key bundle detection from to device events; const encryptedRooms = this.filterEncryptedRoomResponse(data.rooms?.join ?? {}); diff --git a/src/client/call/CallSession.ts b/src/client/call/CallSession.ts new file mode 100644 index 00000000..689f8677 --- /dev/null +++ b/src/client/call/CallSession.ts @@ -0,0 +1,282 @@ +import { Disposable, Disposer } from 'neurone/lib/utils/disposable'; +import { ObservableValue } from 'neurone/lib/utils/observable'; +import { StateEventWithoutRoomId } from 'neurone/lib/api/types'; +import { ObjectReader } from 'neurone/lib/utils'; +import { Storage } from '../storage'; +import { + JoinRoomStateObject, + JoinRoomStickyObject, + JoinRoomMemberObject, + getJoinRoomMemberId, +} from '../storage/collections'; +import { CALL_APPLICATION_TYPE, DEFAULT_CALL_SLOT_IDS, resolveActiveRtcMembers } from './rtcMember'; + +export type CallMember = { + kind: 'call.member' | 'rtc.member'; + /** Unique within the room. */ + id: string; + userId: string; + deviceId?: string; + expiresAt: number; + intent?: 'audio' | 'video'; + capabilities?: string[]; + event: StateEventWithoutRoomId | JoinRoomStickyObject['event']; +}; + +// js-sdk's DEFAULT_EXPIRE_DURATION, used when `expires` is missing. +const DEFAULT_CALL_MEMBER_EXPIRE_MS = 4 * 60 * 60 * 1000; + +/** + * Absolute expiry (unix ms) of a legacy `call.member` event, or undefined if the + * event doesn't represent an active membership (empty content = left). + */ +export const getCallMemberExpiresAt = (event: StateEventWithoutRoomId): number | undefined => { + const { content } = event; + if (!content || Object.keys(content).length === 0) return undefined; + + // Absolute timestamp variants, if an implementation sends them. + const absolute = + ObjectReader.number(content, 'expires_ts') ?? ObjectReader.number(content, 'expires_at'); + if (absolute !== undefined) return absolute; + + // `expires` is a lifetime in ms, counted from the join time. + const startTs = + ObjectReader.number(content, 'created_ts') ?? ObjectReader.number(event, 'origin_server_ts'); + if (startTs === undefined) return undefined; + + return startTs + (ObjectReader.number(content, 'expires') ?? DEFAULT_CALL_MEMBER_EXPIRE_MS); +}; + +export const isActiveCallMember = (event: StateEventWithoutRoomId, now: number): boolean => { + const expiresAt = getCallMemberExpiresAt(event); + return expiresAt !== undefined && expiresAt > now; +}; + +type EventLike = { event: { event_id: string } }; + +const isSameEventIds = (prev: EventLike[], next: EventLike[]): boolean => { + if (prev.length !== next.length) return false; + const prevIds = new Set(prev.map((o) => o.event.event_id)); + const nextIds = new Set(next.map((o) => o.event.event_id)); + return prevIds.size === nextIds.size && [...nextIds].every((id) => prevIds.has(id)); +}; + +type MemberEntries = ObservableValue>; + +export class CallSession implements Disposable { + public readonly roomId: string; + + /** Active call members (legacy + MatrixRTC) whose room membership is `join`. */ + public readonly members: ObservableValue; + + private readonly storage: Storage; + + private readonly publisherKey = crypto.randomUUID(); + + private callEvents: JoinRoomStateObject[] = []; + + private rtcEvents: JoinRoomStickyObject[] = []; + + private slotEvents: JoinRoomStateObject[] = []; + + private readonly redacted = new ObservableValue>(new Set(), this.publisherKey); + + /** Room-member entries for the current senders. */ + private memberEntries?: MemberEntries; + + private memberEntriesDisposer?: Disposer; + + private sendersKey = ''; + + /** Bumped whenever the sender set changes so stale async work is discarded. */ + private generation = 0; + + private expiryTimer?: ReturnType; + + private disposed = false; + + private disposer = new Disposer(); + + constructor(roomId: string, storage: Storage) { + this.roomId = roomId; + this.storage = storage; + this.members = new ObservableValue([], this.publisherKey); + + this.disposer.collect( + this.redacted.subscribe(() => { + this.refresh(); + }) + ); + } + + public async handleRedactions(eventIds: string[]) { + if (eventIds.length === 0) return; + const redacted = new Set([...this.redacted.read, ...eventIds]); + this.redacted.publish(this.publisherKey, redacted); + } + + public async handleCallMemberChange(events: JoinRoomStateObject[]) { + if (isSameEventIds(this.callEvents, events)) return; + this.callEvents = events; + await this.refresh(); + } + + public async handleRtcMemberChange(events: JoinRoomStickyObject[]) { + if (isSameEventIds(this.rtcEvents, events)) return; + this.rtcEvents = events; + await this.refresh(); + } + + public async handleRtcSlotChange(events: JoinRoomStateObject[]) { + if (isSameEventIds(this.slotEvents, events)) return; + this.slotEvents = events; + await this.refresh(); + } + + private async refresh() { + if (this.disposed) return; + await this.syncMemberEntries(); + this.recompute(); + } + + /** Slot ids of open default call slots (`m.call#room`). */ + private getOpenSlotIds(): Set { + const open = new Set(); + this.slotEvents.forEach(({ event }) => { + const application = ObjectReader.object(event.content, 'application'); + if ( + DEFAULT_CALL_SLOT_IDS.has(event.state_key) && + ObjectReader.string(event.content, 'status') === 'open' && + ObjectReader.string(application, 'type') === CALL_APPLICATION_TYPE + ) { + open.add(event.state_key); + } + }); + return open; + } + + private getActiveRtcMembers(now: number) { + const openSlots = this.getOpenSlotIds(); + if (openSlots.size === 0) return []; + return resolveActiveRtcMembers( + this.rtcEvents.map((obj) => obj.event), + now + ).filter((m) => openSlots.has(m.slotId)); + } + + private collectSenders(): string[] { + const now = Date.now(); + const senders = new Set(); + this.callEvents.forEach(({ event }) => { + if (isActiveCallMember(event, now)) senders.add(event.sender); + }); + this.getActiveRtcMembers(now).forEach((m) => senders.add(m.userId)); + return [...senders].sort(); + } + + private async syncMemberEntries() { + const senders = this.collectSenders(); + const key = senders.join('\n'); + if (key === this.sendersKey) return; + + this.sendersKey = key; + this.generation += 1; + const { generation } = this; + + if (senders.length === 0) { + const old = this.memberEntriesDisposer; + this.memberEntries = undefined; + this.memberEntriesDisposer = undefined; + old?.run(); + return; + } + + const entries = await this.storage.joinRoomMember.observableRead.getMultiple( + senders.map((userId) => getJoinRoomMemberId(this.roomId, userId)) + ); + + // A newer sender set arrived (or we were disposed) while awaiting. + if (this.disposed || generation !== this.generation) { + entries.dispose(); + return; + } + + // Keep the old entries until the new ones are ready, so `members` doesn't flicker. + const old = this.memberEntriesDisposer; + const disposer = new Disposer(); + disposer.collect( + entries.subscribe(() => this.recompute()), + entries // disposed exactly once, via the disposer + ); + this.memberEntries = entries; + this.memberEntriesDisposer = disposer; + old?.run(); + } + + private recompute() { + if (this.disposed) return; + + const now = Date.now(); + + const joined = new Set(); + this.memberEntries?.read.forEach((entry) => { + if (entry && ObjectReader.string(entry.event.content, 'membership') === 'join') { + joined.add(entry.userId); + } + }); + + const next: CallMember[] = []; + let nextExpiry = Infinity; + + this.callEvents.forEach(({ event }) => { + const expiresAt = getCallMemberExpiresAt(event); + if (expiresAt === undefined || expiresAt <= now) return; + + nextExpiry = Math.min(nextExpiry, expiresAt); + if (!joined.has(event.sender)) return; + + next.push({ + kind: 'call.member', + id: `call.member:${event.state_key}`, + userId: event.sender, + expiresAt, + event, + }); + }); + + this.getActiveRtcMembers(now).forEach((m) => { + if (this.redacted.read.has(m.event.event_id)) { + return; + } + nextExpiry = Math.min(nextExpiry, m.expiresAt); + if (!joined.has(m.userId)) return; + + next.push({ + kind: 'rtc.member', + id: `rtc.member:${m.userId}:${m.memberId}`, + userId: m.userId, + deviceId: m.deviceId, + expiresAt: m.expiresAt, + intent: m.intent, + capabilities: m.capabilities, + event: m.event, + }); + }); + + this.members.publish(this.publisherKey, next); + + // Expiry produces no storage event, so wake up at the next one. + clearTimeout(this.expiryTimer); + if (Number.isFinite(nextExpiry)) { + this.expiryTimer = setTimeout(() => this.recompute(), nextExpiry - now + 50); + } + } + + dispose(): void { + if (this.disposed) return; + this.disposed = true; + clearTimeout(this.expiryTimer); + this.memberEntriesDisposer?.run(); + this.members.dispose(); + } +} diff --git a/src/client/call/CallSessions.ts b/src/client/call/CallSessions.ts new file mode 100644 index 00000000..e7793028 --- /dev/null +++ b/src/client/call/CallSessions.ts @@ -0,0 +1,129 @@ +import { Disposable, Disposer } from 'neurone/lib/utils/disposable'; +import { ObservableValue } from 'neurone/lib/utils/observable'; +import { Storage } from '../storage'; +import { JoinRoomStateObject, JoinRoomStickyObject } from '../storage/collections'; +import { CallSession } from './CallSession'; +import { RTC_MEMBER_EVENT_TYPE, RTC_SLOT_EVENT_TYPE } from './rtcMember'; +import { SyncValue } from '../sync'; +import { rContent, rEvent } from '../reader'; + +export class CallSessions implements Disposable { + private readonly storage: Storage; + + private readonly syncObservable: ObservableValue; + + private readonly disposer = new Disposer(); + + private readonly roomToSessions = new Map(); + + constructor(storage: Storage, syncObservable: ObservableValue) { + this.storage = storage; + this.syncObservable = syncObservable; + + this.start(); + } + + private async start() { + const callMemberStateEvents = await this.storage.joinRoomState.observableRead.queryIndex( + 'type', + 'org.matrix.msc3401.call.member' + ); + + const rtcSlotEvents = await this.storage.joinRoomState.observableRead.queryIndex( + 'type', + RTC_SLOT_EVENT_TYPE + ); + + const rtcMemberEvents = await this.storage.joinRoomSticky.observableRead.queryIndex( + 'type', + RTC_MEMBER_EVENT_TYPE + ); + + // Slots first, so sessions know whether the slot is open before members arrive. + this.handleRtcSlots(rtcSlotEvents.read); + this.handleCallMembers(callMemberStateEvents.read); + this.handleRtcMembers(rtcMemberEvents.read); + + this.disposer.collect( + callMemberStateEvents.subscribe((objs) => this.handleCallMembers(objs)), + callMemberStateEvents, + rtcSlotEvents.subscribe((objs) => this.handleRtcSlots(objs)), + rtcSlotEvents, + rtcMemberEvents.subscribe((objs) => this.handleRtcMembers(objs)), + rtcMemberEvents, + this.syncObservable.subscribe((value) => this.handleSync(value)) + ); + } + + private handleSync(syncValue: SyncValue): void { + const [, result] = syncValue ?? []; + Object.keys(result?.rooms?.join ?? {}).forEach((roomId) => { + const joinRoom = result?.rooms?.join?.[roomId]; + if (!joinRoom) return; + const redactionEvents = joinRoom.timeline?.events.filter( + (event) => event.type === 'm.room.redaction' + ); + if (!redactionEvents || redactionEvents.length === 0) return; + + const eventIds = redactionEvents.map( + (event) => rEvent.redacts(event) ?? rContent.redacts(event.content) + ); + + const session = this.roomToSessions.get(roomId); + session?.handleRedactions(eventIds.filter((id) => typeof id === 'string')); + }); + } + + /** + * Groups events per room and hands them to the matching session. + * Rooms that already have a session but no events are given an empty list, + * otherwise their members would never clear. + */ + private dispatch( + objs: T[], + apply: (session: CallSession, events: T[]) => Promise + ) { + const byRoom = new Map(); + objs.forEach((obj) => { + const list = byRoom.get(obj.roomId) ?? []; + list.push(obj); + byRoom.set(obj.roomId, list); + }); + + // Existing sessions whose room has NO events in this update get an empty list + this.roomToSessions.forEach((_, roomId) => { + if (!byRoom.has(roomId)) byRoom.set(roomId, []); + }); + + byRoom.forEach((events, roomId) => { + apply(this.getSession(roomId), events).catch((e) => console.error(e)); + }); + } + + private handleCallMembers(objs: JoinRoomStateObject[]): void { + this.dispatch(objs, (s, e) => s.handleCallMemberChange(e)); + } + + private handleRtcSlots(objs: JoinRoomStateObject[]): void { + this.dispatch(objs, (s, e) => s.handleRtcSlotChange(e)); + } + + private handleRtcMembers(objs: JoinRoomStickyObject[]): void { + this.dispatch(objs, (s, e) => s.handleRtcMemberChange(e)); + } + + public getSession(roomId: string): CallSession { + let session = this.roomToSessions.get(roomId); + if (!session) { + session = new CallSession(roomId, this.storage); + this.roomToSessions.set(roomId, session); + } + return session; + } + + dispose(): void { + this.roomToSessions.forEach((s) => s.dispose()); + this.roomToSessions.clear(); + this.disposer.run(); + } +} diff --git a/src/client/call/index.ts b/src/client/call/index.ts new file mode 100644 index 00000000..37084d6c --- /dev/null +++ b/src/client/call/index.ts @@ -0,0 +1,2 @@ +export * from './CallSessions'; +export * from './CallSession'; diff --git a/src/client/call/rtcMember.ts b/src/client/call/rtcMember.ts new file mode 100644 index 00000000..10dbd1cf --- /dev/null +++ b/src/client/call/rtcMember.ts @@ -0,0 +1,144 @@ +import { ObjectReader } from 'neurone/lib/utils'; +import { JoinRoomStickyObject } from '../storage/collections'; + +type StickyEvent = JoinRoomStickyObject['event']; + +// Unstable identifiers for now; add the stable ones here once MSC4143 lands. +export const RTC_SLOT_EVENT_TYPE = 'org.matrix.msc4143.rtc.slot'; +export const RTC_MEMBER_EVENT_TYPE = 'org.matrix.msc4143.rtc.member'; + +export const CALL_APPLICATION_TYPE = 'm.call'; +// MSC4196 says `room`, the reference implementation uses `ROOM`. Accept both. +export const DEFAULT_CALL_SLOT_IDS: ReadonlySet = new Set(['m.call#room', 'm.call#ROOM']); + +const STICKY_MAX_DURATION_MS = 60 * 60 * 1000; + +export type RtcMember = { + stickyKey: string; + memberId: string; + userId: string; + deviceId?: string; + slotId: string; + expiresAt: number; + intent?: 'audio' | 'video'; + capabilities?: string[]; + event: StickyEvent; +}; + +type ResolvedEntry = { + stickyKey: string; + memberId?: string; + membership?: string; + slotId?: string; + applicationType?: string; + deviceId?: string; + intent?: 'audio' | 'video'; + capabilities?: string[]; + expiresAt: number; + eventId: string; + sender: string; + event: StickyEvent; +}; + +export const getStickyExpiresAt = (event: StickyEvent): number | undefined => { + const sticky = + ObjectReader.object(event, 'msc4354_sticky') ?? ObjectReader.object(event, 'sticky'); + const duration = sticky ? ObjectReader.number(sticky, 'duration_ms') : undefined; + const originTs = ObjectReader.number(event, 'origin_server_ts'); + if (duration === undefined || originTs === undefined) return undefined; + return originTs + Math.min(duration, STICKY_MAX_DURATION_MS); +}; + +const parseIntent = (application: object | undefined): 'audio' | 'video' | undefined => { + if (!application) return undefined; + // Spec: `intent: m.audio|m.video`. Reference impl: `m.call.intent: audio|video`. + const raw = + ObjectReader.string(application, 'intent') ?? + ObjectReader.string(application, 'm.call.intent') ?? + ObjectReader.string(ObjectReader.object(application, 'm.call'), 'intent'); + if (raw === 'm.audio' || raw === 'audio') return 'audio'; + if (raw === 'm.video' || raw === 'video') return 'video'; + return undefined; +}; + +const parseEntry = (event: StickyEvent): ResolvedEntry | undefined => { + const { content } = event; + const expiresAt = getStickyExpiresAt(event); + if (!content || expiresAt === undefined) return undefined; + + const member = ObjectReader.object(content, 'member'); + const application = ObjectReader.object(content, 'application'); + const memberId = ObjectReader.string(member, 'id'); + + const stickyKey = + ObjectReader.string(content, 'sticky_key') ?? + ObjectReader.string(content, 'msc4354_sticky_key') ?? + memberId; + if (!stickyKey) return undefined; // not part of an ephemeral map + + return { + stickyKey, + memberId, + membership: ObjectReader.string(member, 'membership'), + slotId: ObjectReader.string(content, 'slot_id'), + applicationType: ObjectReader.string(application, 'type'), + deviceId: ObjectReader.string(member, 'device_id'), + intent: parseIntent(application), + capabilities: ObjectReader.array(application, 'capabilities')?.filter( + (c): c is string => typeof c === 'string' + ), + expiresAt, + eventId: event.event_id, + sender: event.sender, + event, + }; +}; + +// MSC4354 tie-break: highest expiry wins, then highest event ID. +const isNewer = (a: ResolvedEntry, b: ResolvedEntry): boolean => + a.expiresAt !== b.expiresAt ? a.expiresAt > b.expiresAt : a.eventId > b.eventId; + +/** + * Builds the MSC4354 ephemeral map for one room and returns the entries that are + * currently joined to the default call slot and not expired. + * Slot-open and room-membership checks are done by the caller. + */ +export const resolveActiveRtcMembers = (events: StickyEvent[], now: number): RtcMember[] => { + const winners = new Map(); + + events.forEach((event) => { + if (event.type !== RTC_MEMBER_EVENT_TYPE) return; + const entry = parseEntry(event); + if (!entry) return; + + const mapKey = `${entry.sender}\n${entry.stickyKey}`; + const current = winners.get(mapKey); + if (!current || isNewer(entry, current)) winners.set(mapKey, entry); + }); + + const active: RtcMember[] = []; + winners.forEach((entry) => { + if ( + entry.membership !== 'join' || + entry.expiresAt <= now || + !entry.memberId || + !entry.slotId || + !DEFAULT_CALL_SLOT_IDS.has(entry.slotId) || + entry.applicationType !== CALL_APPLICATION_TYPE + ) { + return; + } + active.push({ + stickyKey: entry.stickyKey, + memberId: entry.memberId, + userId: entry.sender, + deviceId: entry.deviceId, + slotId: entry.slotId, + expiresAt: entry.expiresAt, + intent: entry.intent, + capabilities: entry.capabilities, + event: entry.event, + }); + }); + return active; +}; diff --git a/src/client/context.ts b/src/client/context.ts index 791d3191..4dde5291 100644 --- a/src/client/context.ts +++ b/src/client/context.ts @@ -13,6 +13,7 @@ import { StateEventWithoutRoomId, } from 'neurone/lib/api/types'; import { AesHmacSha2EncryptedData, SecretStorage } from 'neurone/lib/crypto/ssss'; +import { ProcessedToDeviceEvent } from 'neurone/lib/crypto/machine/events'; import { InviteRoomStateObject, JoinRoomStateObject, @@ -30,6 +31,7 @@ import { Timeline } from './room'; import { AccountDataType } from '../types/matrix/accountData'; import { DecryptionManager } from './decryption'; import { MegolmBackup } from './MegolmBackup'; +import { CallSessions } from './call'; export type StateContext = { join: { @@ -123,6 +125,8 @@ export type ClientContext = { pendingMemberRooms: ObservableValue; verificationRequest: Observable; + toDeviceEvents: Observable; + callSessions: CallSessions; secrets: Secrets; backupInfo: ObservableValue; megolmBackup: MegolmBackup; @@ -939,6 +943,20 @@ export const createVerificationRequestObservable = () => { }; }; +export const createToDeviceEventsObservable = () => { + const publishKey = crypto.randomUUID(); + const toDeviceEvents = new Observable(publishKey); + + const onToDeviceEvents = (events: ProcessedToDeviceEvent[]) => { + toDeviceEvents.publish(publishKey, events); + }; + + return { + toDeviceEvents, + onToDeviceEvents, + }; +}; + export const createSecretsObservable = async ( storage: Storage, disposer: Disposer diff --git a/src/client/decryption/DecryptionManager.ts b/src/client/decryption/DecryptionManager.ts index b1867144..1b37844f 100644 --- a/src/client/decryption/DecryptionManager.ts +++ b/src/client/decryption/DecryptionManager.ts @@ -6,7 +6,7 @@ import { import { Machine, RoomKeyInfo, RoomKeyWithheldInfo } from 'neurone/lib/crypto/machine'; import { ObjectReader } from 'neurone/lib/utils'; import { Disposable, Disposer } from 'neurone/lib/utils/disposable'; -import { EncryptedEvent } from './EncryptedEvent'; +import { EncryptedEvent, EventDecryptionSuccess } from './EncryptedEvent'; import { EncryptedEventIndex } from './EncryptedEventIndex'; export class DecryptionManager implements Disposable { @@ -79,6 +79,35 @@ export class DecryptionManager implements Disposable { return this.registerEvent(this.stateEvents, roomId, event); } + /** + * Attempts to decrypt an encrypted room event. + * If the event is not encrypted or decryption fails, returns the original event unchanged. + * @param roomId - The ID of the room the event belongs to. + * @param event - The event to decrypt. + * @returns A promise that resolves to the decrypted event if decryption succeeds, + * or the original event otherwise. + */ + public async tryDecryptEvent( + roomId: string, + event: ClientEventWithoutRoomId + ): Promise { + if (event.type !== 'm.room.encryption') { + return event; + } + const encryptedEvent = this.registerEvent( + 'state_key' in event ? this.stateEvents : this.messageEvents, + roomId, + event + ); + await encryptedEvent.decrypt(); + const decryption = encryptedEvent.decryption.read; + if (decryption instanceof EventDecryptionSuccess) { + return decryption.event; + } + + return event; + } + dispose(): void { this.disposer.run(); } diff --git a/src/client/initialize.ts b/src/client/initialize.ts index fc520e32..62c8b0c4 100644 --- a/src/client/initialize.ts +++ b/src/client/initialize.ts @@ -26,6 +26,7 @@ import { createVerificationRequestObservable, createBackupInfoObservable, createUserReceiptObservable, + createToDeviceEventsObservable, } from './context'; import { createStorage } from './storage'; import { getAuthJSONRequest } from './request'; @@ -39,6 +40,7 @@ import { DecryptionManager } from './decryption'; import { initializeCrypto } from './crypto'; import { MegolmBackup } from './MegolmBackup'; import { clearUserLocalStorage } from './localStorage'; +import { CallSessions } from './call'; export type InitializeClientResult = { ct: ClientContext; @@ -85,6 +87,7 @@ export const initializeClient = async (session: Session): Promise new Collection(ROOM_STATE_STORE_NAME, 'id', driver); +/** + * JoinRoomSticky Store + */ +const ROOM_STATE_STICKY_NAME = 'JoinRoomSticky'; + +export const prepareJoinRoomStickyStore = (idb: IDBPDatabase, oldVersion: number) => { + if (oldVersion < 3) { + const store = idb.createObjectStore(ROOM_STATE_STICKY_NAME, { + keyPath: 'id', + }); + + store.createIndex('roomId', 'roomId', { unique: false }); + store.createIndex('type', 'type', { unique: false }); + store.createIndex('expiresAt', 'expiresAt', { unique: false }); + } +}; + +export type JoinRoomStickyObject = Entry< + 'id', + 'roomId' | 'type' | 'expiresAt', + { + id: string; + roomId: string; + type: string; + expiresAt: number; + event: MessageEventWithoutRoomId; + } +>; + +export type JoinRoomStickyCollection = Collection< + 'id', + 'roomId' | 'type' | 'expiresAt', + JoinRoomStickyObject +>; + +export const createJoinRoomStickyCollection = (driver: StorageDriver): JoinRoomStickyCollection => + new Collection(ROOM_STATE_STICKY_NAME, 'id', driver); + /** * JoinRoomMember Store */ diff --git a/src/client/storage/index.ts b/src/client/storage/index.ts index b69edc6e..81310c71 100644 --- a/src/client/storage/index.ts +++ b/src/client/storage/index.ts @@ -46,12 +46,15 @@ import { JoinRoomLatestCollection, createJoinRoomLatestEventCollection, prepareJoinRoomLatestEventStore, + prepareJoinRoomStickyStore, + JoinRoomStickyCollection, + createJoinRoomStickyCollection, } from './collections'; import { getSessionStoreName, Session } from '../sessions'; export * from './utils'; -const DATABASE_VERSION = 2; +const DATABASE_VERSION = 3; export interface Storage { sessionValue: SessionValueCollection; @@ -61,6 +64,7 @@ export interface Storage { joinAccountData: JoinAccountDataCollection; joinRoomSummary: JoinRoomSummaryCollection; joinRoomState: JoinRoomStateCollection; + joinRoomSticky: JoinRoomStickyCollection; joinRoomMember: JoinRoomMemberCollection; joinRoomLatestEvent: JoinRoomLatestCollection; @@ -102,6 +106,7 @@ export const createStorage = async ( prepareJoinAccountDataStore(db, oldVersion); prepareJoinRoomSummaryStore(db, oldVersion); prepareJoinRoomStateStore(db, oldVersion); + prepareJoinRoomStickyStore(db, oldVersion); prepareJoinRoomMemberStore(db, oldVersion); prepareJoinRoomLatestEventStore(db, oldVersion); @@ -138,6 +143,7 @@ export const createStorage = async ( joinAccountData: createJoinAccountDataCollection(driver), joinRoomSummary: createJoinRoomSummaryCollection(driver), joinRoomState: createJoinRoomStateCollection(driver), + joinRoomSticky: createJoinRoomStickyCollection(driver), joinRoomMember: createJoinRoomMemberCollection(driver), joinRoomLatestEvent: createJoinRoomLatestEventCollection(driver), diff --git a/src/client/syncWriter/index.ts b/src/client/syncWriter/index.ts index fab0462c..26b92b3c 100644 --- a/src/client/syncWriter/index.ts +++ b/src/client/syncWriter/index.ts @@ -24,6 +24,7 @@ import { writeReadReceipt, writeUnreadNotifications, writeUnreadThreadNotifications, + writeJoinRoomSticky, } from './utils'; import { sessionValue } from '../storage/collections'; @@ -60,6 +61,7 @@ export const writeSync = async (ct: ClientContext, data: GetSyncOK): Promise +): Promise => { + const expiredEventKeys = await ct.storage.joinRoomSticky.read.queryIndexKeys( + 'expiresAt', + new QueryRange(0, Date.now()) + ); + if (expiredEventKeys.length > 0) { + await ct.storage.joinRoomSticky.write.deleteMultiple(expiredEventKeys); + } + + const stickyEventObjects: JoinRoomStickyObject[] = joinRooms.flatMap((roomId) => { + const joinRoom = join[roomId]; + + const stickyEvents = Array.from( + (ObjectReader.array(ObjectReader.object(joinRoom, 'msc4354_sticky'), 'events') as + | MessageEventWithoutRoomId[] + | undefined) ?? [] + ); + + stickyEvents.push( + ...joinRoomUtil.roomTimeline(joinRoom).filter((event) => { + const remainingMs = ObjectReader.number(event.unsigned, 'msc4354_sticky_duration_ttl_ms'); + return remainingMs && remainingMs > 0; + }) + ); + + const stickyObjects: JoinRoomStickyObject[] = []; + + stickyEvents.forEach((event) => { + const remainingMs = ObjectReader.number(event.unsigned, 'msc4354_sticky_duration_ttl_ms'); + if (!remainingMs || remainingMs <= 0) return; + + const obj: JoinRoomStickyObject = { + id: event.event_id, + roomId, + type: event.type, + expiresAt: Date.now() + remainingMs, + event, + }; + + stickyObjects.push(obj); + }); + + return stickyObjects; + }); + + await ct.storage.joinRoomSticky.write.putMultiple(stickyEventObjects); +}; + export const writeJoinRoomMember = async ( ct: ClientContext, joinRooms: string[], diff --git a/src/types/matrix/room.ts b/src/types/matrix/room.ts index 3984a988..174a6858 100644 --- a/src/types/matrix/room.ts +++ b/src/types/matrix/room.ts @@ -76,6 +76,22 @@ export enum MessageEvent { Reaction = 'm.reaction', } +export enum CallEventType { + CallInvite = 'm.call.invite', + CallCandidates = 'm.call.candidates', + CallAnswer = 'm.call.answer', + CallHangup = 'm.call.hangup', + CallReject = 'm.call.reject', + CallSelectAnswer = 'm.call.select_answer', + CallNegotiate = 'm.call.negotiate', + CallSDPStreamMetadataChanged = 'm.call.sdp_stream_metadata_changed', + CallSDPStreamMetadataChangedPrefix = 'org.matrix.call.sdp_stream_metadata_changed', + CallReplaces = 'm.call.replaces', + CallEncryptionKeysPrefix = 'io.element.call.encryption_keys', + RTCDecline = 'org.matrix.msc4310.rtc.decline', + RTCInvite = 'org.matrix.msc4075.rtc.notification', +} + export enum RoomType { Space = 'm.space', Call = 'org.matrix.msc3417.call', diff --git a/vite.config.js b/vite.config.js index a5af94d8..2c2e59ef 100644 --- a/vite.config.js +++ b/vite.config.js @@ -17,6 +17,11 @@ const copyFiles = { src: 'node_modules/@element-hq/element-call-embedded/dist/*', dest: 'public/element-call', }, + { + src: 'call.config.json', + dest: 'public/element-call', + rename: 'config.json', + }, { src: 'node_modules/pdfjs-dist/build/pdf.worker.min.mjs', dest: '',