diff --git a/README.md b/README.md index 9b660ae..2a2b0f2 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,8 @@ the installed Android SDK and CultLib's `cultmesh-kotlin` package, without Gradle or Flutter. It exists to put Eve on Periwinkle immediately: it consumes typed `mimir.eve_dashboard_state.v1` CultMesh documents, sends `mimir.eve_dashboard_command.v1` command documents, and publishes timestamped -touch/motion observations as `mimir.eve_sensor_observation.v1`. +touch/motion observations as `mimir.eve_sensor_observation.v1` plus camera/mic +payloads as `mimir.eve_media_observation.v1`. - `EVEAppDelegate` creates one fullscreen `UIWindow`. - `EVEViewController` installs: @@ -56,13 +57,13 @@ touch/motion observations as `mimir.eve_sensor_observation.v1`. - `EVEFrameStreamClient` receives binary JPEG frames from the Starfire CEF relay and sends touch events back as JSON viewport coordinates. - `EVESensorUplinkClient` opens separate WebSocket uplinks for camera and - microphone frame-events so sensor traffic does not block display/control + microphone observations so sensor traffic does not block display/control traffic. - `EVEDashboardClient` opens the native Mimir dashboard socket and receives scene/control state snapshots. - `EVEViewController` captures camera frames with AVFoundation and microphone - blocks with AVAudioEngine, then sends `eve-camera` and `eve-mic` samples to - Mimir. It also renders the dashboard natively with UIKit: a scene graph, + blocks with AVAudioEngine, then sends binary CultMesh `eve-camera` and + `eve-mic` media observations to Mimir. It also renders the dashboard natively with UIKit: a scene graph, draggable source panels, visibility/reset controls, and multitouch pan/pinch/rotate transform commands. @@ -131,11 +132,10 @@ dotnet run --project E:\Projects\Mimir\src\Mimir.EveSensorReceiver\Mimir.EveSens dotnet run --project E:\Projects\Mimir\src\Mimir.EveSensorReceiver\Mimir.EveSensorReceiver.csproj -- --port 8794 --path /eve/mic --source-id eve-mic --type audio-block ``` -EveCanvas sends camera frame-events to `ws://192.168.1.66:8793/eve/camera` and -microphone frame-events to `ws://192.168.1.66:8794/eve/mic`. The first transport -uses JSON plus base64 payloads because it is inspectable and already matches -Mimir's frame-event source. Replace it with binary framing only after the -sample contract is proven on device. +EveCanvas sends binary `mimir.eve_media_observation.v1` camera observations to +`ws://192.168.1.66:8793/eve/camera` and microphone observations to +`ws://192.168.1.66:8794/eve/mic`. Periwinkle sends camera, microphone, motion, +and touch observations through `ws://192.168.1.66:8796/eve/periwinkle`. ## Native Mimir Dashboard diff --git a/Sources/EVESensorUplinkClient.h b/Sources/EVESensorUplinkClient.h index ec1f863..efbd37a 100644 --- a/Sources/EVESensorUplinkClient.h +++ b/Sources/EVESensorUplinkClient.h @@ -6,6 +6,7 @@ - (void)connect; - (void)disconnect; - (void)sendJSONObject:(NSDictionary *)payload; +- (void)sendData:(NSData *)payload; @property(nonatomic, copy, readonly) NSString *status; diff --git a/Sources/EVESensorUplinkClient.m b/Sources/EVESensorUplinkClient.m index 64aae12..9672525 100644 --- a/Sources/EVESensorUplinkClient.m +++ b/Sources/EVESensorUplinkClient.m @@ -88,6 +88,37 @@ }]; } +- (void)sendData:(NSData *)payload { + NSURLSessionWebSocketTask *task = self.task; + if (!task) { + [self connect]; + return; + } + + if (!payload) { + self.status = [NSString stringWithFormat:@"%@ binary missing", self.label]; + return; + } + + NSURLSessionWebSocketMessage *message = [[NSURLSessionWebSocketMessage alloc] initWithData:payload]; + __weak typeof(self) weakSelf = self; + [task sendMessage:message completionHandler:^(NSError *error) { + __strong typeof(weakSelf) self = weakSelf; + if (!self) { + return; + } + + if (error) { + self.status = [NSString stringWithFormat:@"%@ send failed", self.label]; + NSLog(@"EveCanvas %@ uplink binary send failed: %@", self.label, error); + self.task = nil; + [self reconnectSoon]; + } else { + self.status = [NSString stringWithFormat:@"%@ live", self.label]; + } + }]; +} + - (void)reconnectSoon { if (!self.shouldReconnect) { return; diff --git a/Sources/EVEViewController.mm b/Sources/EVEViewController.mm index 16731cb..52cbd57 100644 --- a/Sources/EVEViewController.mm +++ b/Sources/EVEViewController.mm @@ -15,6 +15,117 @@ static int64_t EVEHostTimeNowNs(void) { return (int64_t)(CACurrentMediaTime() * 1000000000.0); } +static void EVEAppendByte(NSMutableData *data, uint8_t value) { + [data appendBytes:&value length:1]; +} + +static void EVEAppendUInt16(NSMutableData *data, uint16_t value) { + uint8_t bytes[2] = { (uint8_t)((value >> 8) & 0xff), (uint8_t)(value & 0xff) }; + [data appendBytes:bytes length:2]; +} + +static void EVEAppendUInt32(NSMutableData *data, uint32_t value) { + uint8_t bytes[4] = { + (uint8_t)((value >> 24) & 0xff), + (uint8_t)((value >> 16) & 0xff), + (uint8_t)((value >> 8) & 0xff), + (uint8_t)(value & 0xff), + }; + [data appendBytes:bytes length:4]; +} + +static void EVEAppendInt64(NSMutableData *data, int64_t value) { + EVEAppendByte(data, 0xd3); + uint64_t raw = (uint64_t)value; + uint8_t bytes[8] = { + (uint8_t)((raw >> 56) & 0xff), + (uint8_t)((raw >> 48) & 0xff), + (uint8_t)((raw >> 40) & 0xff), + (uint8_t)((raw >> 32) & 0xff), + (uint8_t)((raw >> 24) & 0xff), + (uint8_t)((raw >> 16) & 0xff), + (uint8_t)((raw >> 8) & 0xff), + (uint8_t)(raw & 0xff), + }; + [data appendBytes:bytes length:8]; +} + +static void EVEAppendNullableInt64(NSMutableData *data, NSNumber *value) { + if (!value) { + EVEAppendByte(data, 0xc0); + return; + } + + EVEAppendInt64(data, value.longLongValue); +} + +static void EVEAppendString(NSMutableData *data, NSString *value) { + NSData *bytes = [(value ?: @"") dataUsingEncoding:NSUTF8StringEncoding] ?: [NSData data]; + NSUInteger length = bytes.length; + if (length < 32) { + EVEAppendByte(data, (uint8_t)(0xa0 | length)); + } else if (length < 256) { + EVEAppendByte(data, 0xd9); + EVEAppendByte(data, (uint8_t)length); + } else { + EVEAppendByte(data, 0xda); + EVEAppendUInt16(data, (uint16_t)MIN(length, 65535)); + } + [data appendData:bytes]; +} + +static void EVEAppendBinary(NSMutableData *data, NSData *payload) { + NSUInteger length = payload.length; + if (length < 256) { + EVEAppendByte(data, 0xc4); + EVEAppendByte(data, (uint8_t)length); + } else if (length <= 65535) { + EVEAppendByte(data, 0xc5); + EVEAppendUInt16(data, (uint16_t)length); + } else { + EVEAppendByte(data, 0xc6); + EVEAppendUInt32(data, (uint32_t)length); + } + [data appendData:payload ?: [NSData data]]; +} + +static NSData *EVEMediaObservation(NSString *observationId, + NSString *deviceId, + NSString *streamId, + NSString *kind, + uint64_t sequence, + int64_t sensorTimestampNs, + int64_t elapsedRealtimeNs, + NSString *format, + NSNumber *width, + NSNumber *height, + NSNumber *sampleRate, + NSNumber *channels, + NSNumber *frameCount, + NSData *payload) { + NSMutableData *data = [NSMutableData data]; + EVEAppendByte(data, 0xdc); + EVEAppendUInt16(data, 17); + EVEAppendString(data, observationId); + EVEAppendString(data, deviceId); + EVEAppendString(data, streamId); + EVEAppendString(data, kind); + EVEAppendInt64(data, (int64_t)sequence); + EVEAppendInt64(data, sensorTimestampNs); + EVEAppendInt64(data, elapsedRealtimeNs); + EVEAppendString(data, [[NSDate date] descriptionWithLocale:nil]); + EVEAppendString(data, @"eve-host-time"); + EVEAppendString(data, format); + EVEAppendNullableInt64(data, width); + EVEAppendNullableInt64(data, height); + EVEAppendNullableInt64(data, sampleRate); + EVEAppendNullableInt64(data, channels); + EVEAppendNullableInt64(data, frameCount); + EVEAppendString(data, @"raw"); + EVEAppendBinary(data, payload ?: [NSData data]); + return data; +} + @interface EVEViewController () @property(nonatomic, strong) EVEGLView *glView; @@ -418,20 +529,24 @@ static int64_t EVEHostTimeNowNs(void) { } } - NSString *base64 = [samples base64EncodedStringWithOptions:0]; - NSDictionary *payload = @{ - @"type": @"audio-block", - @"sourceId": @"eve-mic", - @"timestampNs": @(EVEHostTimeNowNs()), - @"sequence": @(self.micSequence++), - @"sampleRate": @((int)format.sampleRate), - @"channels": @(channels), - @"sampleFormat": @"Float32", - @"frameCount": @(frameCount), - @"byteLength": @(samples.length), - @"samplesBase64": base64, - }; - [self.micUplink sendJSONObject:payload]; + uint64_t sequence = self.micSequence++; + int64_t timestampNs = EVEHostTimeNowNs(); + NSData *payload = EVEMediaObservation( + [NSString stringWithFormat:@"eve:microphone-float32-block:%llu", sequence], + @"eve", + @"eve-mic", + @"microphone-float32-block", + sequence, + timestampNs, + timestampNs, + @"float32le", + nil, + nil, + @((int)format.sampleRate), + @(channels), + @(frameCount), + samples); + [self.micUplink sendData:payload]; }]; NSError *error = nil; @@ -489,19 +604,24 @@ static int64_t EVEHostTimeNowNs(void) { return; } - NSString *base64 = [jpeg base64EncodedStringWithOptions:0]; - NSDictionary *payload = @{ - @"type": @"video-frame", - @"sourceId": @"eve-camera", - @"timestampNs": @(EVEHostTimeNowNs()), - @"sequence": @(self.cameraSequence++), - @"width": @(width), - @"height": @(height), - @"pixelFormat": @"MJPG", - @"byteLength": @(jpeg.length), - @"samplesBase64": base64, - }; - [self.cameraUplink sendJSONObject:payload]; + uint64_t sequence = self.cameraSequence++; + int64_t timestampNs = EVEHostTimeNowNs(); + NSData *payload = EVEMediaObservation( + [NSString stringWithFormat:@"eve:camera-mjpeg-frame:%llu", sequence], + @"eve", + @"eve-camera", + @"camera-mjpeg-frame", + sequence, + timestampNs, + timestampNs, + @"mjpeg", + @(width), + @(height), + nil, + nil, + @1, + jpeg); + [self.cameraUplink sendData:payload]; } - (CGPoint)streamPointForTouch:(UITouch *)touch { diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index 9826ec1..a4c845f 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -1,6 +1,8 @@ + + , grantResults: IntArray) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults) + if (requestCode == 42) startLocalMedia() + } + override fun dispatchTouchEvent(event: MotionEvent): Boolean { val sequence = ++touchSequence val elapsedNs = SystemClock.elapsedRealtimeNanos() @@ -372,6 +407,204 @@ class MainActivity : Activity(), SensorEventListener { } } + private fun requestMediaPermissionsAndStart() { + val missing = arrayOf(Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO) + .filter { checkSelfPermission(it) != PackageManager.PERMISSION_GRANTED } + .toTypedArray() + if (missing.isNotEmpty()) { + requestPermissions(missing, 42) + return + } + + startLocalMedia() + } + + private fun startLocalMedia() { + if (mediaRunning) return + mediaRunning = true + startMicrophone() + startCamera() + } + + @Suppress("MissingPermission") + private fun startMicrophone() { + if (checkSelfPermission(Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) { + main.post { mediaText.text = "media sensors\nmicrophone permission missing" } + return + } + + workers.execute { + val sampleRate = 16_000 + val minBuffer = AudioRecord.getMinBufferSize(sampleRate, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT) + if (minBuffer <= 0) { + main.post { mediaText.text = "media sensors\nAudioRecord unavailable" } + return@execute + } + + val blockBytes = 3_200 + val record = AudioRecord(MediaRecorder.AudioSource.MIC, sampleRate, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, minBuffer.coerceAtLeast(blockBytes * 2)) + audioRecord = record + val buffer = ByteArray(blockBytes) + record.startRecording() + while (mediaRunning && !Thread.currentThread().isInterrupted) { + val read = record.read(buffer, 0, buffer.size) + if (read > 0) { + val payload = buffer.copyOf(read) + val elapsedNs = SystemClock.elapsedRealtimeNanos() + val sequence = ++mediaSequence + main.post { mediaText.text = "media sensors\nmicrophone pcm16le seq=$sequence bytes=$read rate=$sampleRate" } + sendMediaObservation( + kind = "microphone-pcm16-block", + streamId = "periwinkle-mic", + sequence = sequence, + sensorTimestampNs = elapsedNs, + elapsedNs = elapsedNs, + format = "pcm16le", + sampleRate = sampleRate, + channels = 1, + frameCount = read / 2, + payload = payload) + } + } + } + } + + @Suppress("MissingPermission", "DEPRECATION") + private fun startCamera() { + if (checkSelfPermission(Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) { + main.post { mediaText.text = "media sensors\ncamera permission missing" } + return + } + + val manager = getSystemService(CAMERA_SERVICE) as CameraManager + val cameraId = manager.cameraIdList.firstOrNull { id -> + manager.getCameraCharacteristics(id).get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_BACK + } ?: manager.cameraIdList.firstOrNull() + if (cameraId == null) { + main.post { mediaText.text = "media sensors\ncamera unavailable" } + return + } + + imageReader = ImageReader.newInstance(160, 120, ImageFormat.YUV_420_888, 2).apply { + setOnImageAvailableListener({ reader -> + val image = reader.acquireLatestImage() ?: return@setOnImageAvailableListener + handleCameraImage(image) + }, main) + } + + manager.openCamera(cameraId, object : CameraDevice.StateCallback() { + override fun onOpened(camera: CameraDevice) { + cameraDevice = camera + val reader = imageReader ?: return + val request = camera.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW).apply { + addTarget(reader.surface) + } + camera.createCaptureSession(listOf(reader.surface), object : CameraCaptureSession.StateCallback() { + override fun onConfigured(session: CameraCaptureSession) { + cameraSession = session + session.setRepeatingRequest(request.build(), null, main) + } + + override fun onConfigureFailed(session: CameraCaptureSession) { + mediaText.text = "media sensors\ncamera session configure failed" + } + }, main) + } + + override fun onDisconnected(camera: CameraDevice) { + camera.close() + } + + override fun onError(camera: CameraDevice, error: Int) { + mediaText.text = "media sensors\ncamera error=$error" + camera.close() + } + }, main) + } + + private fun handleCameraImage(image: Image) { + image.use { + val elapsedNs = SystemClock.elapsedRealtimeNanos() + if (elapsedNs - lastCameraSentElapsedNs < 200_000_000L) return + lastCameraSentElapsedNs = elapsedNs + val payload = copyLumaPlane(image) + val sequence = ++mediaSequence + mediaText.text = "media sensors\ncamera y8 seq=$sequence ${image.width}x${image.height} bytes=${payload.size}" + sendMediaObservation( + kind = "camera-luma-frame", + streamId = "periwinkle-camera", + sequence = sequence, + sensorTimestampNs = image.timestamp, + elapsedNs = elapsedNs, + format = "y8", + width = image.width, + height = image.height, + frameCount = 1, + payload = payload) + } + } + + private fun copyLumaPlane(image: Image): ByteArray { + val plane = image.planes[0] + val buffer = plane.buffer + val rowStride = plane.rowStride + val pixelStride = plane.pixelStride + val data = ByteArray(image.width * image.height) + var out = 0 + for (y in 0 until image.height) { + val rowStart = y * rowStride + for (x in 0 until image.width) { + data[out++] = buffer.get(rowStart + x * pixelStride) + } + } + return data + } + + private fun sendMediaObservation( + kind: String, + streamId: String, + sequence: Long, + sensorTimestampNs: Long, + elapsedNs: Long, + format: String, + width: Int? = null, + height: Int? = null, + sampleRate: Int? = null, + channels: Int? = null, + frameCount: Int? = null, + payload: ByteArray, + ) { + val socket = sensorSocket ?: return + val observation = EveMediaObservationDocument( + observationId = "$deviceId:$kind:$sequence", + deviceId = deviceId, + streamId = streamId, + kind = kind, + sequence = sequence, + sensorTimestampNs = sensorTimestampNs, + elapsedRealtimeNs = elapsedNs, + wallClockUtc = utcNow(), + clockDomainId = "periwinkle-elapsed-realtime", + format = format, + width = width, + height = height, + sampleRate = sampleRate, + channels = channels, + frameCount = frameCount, + payloadEncoding = "raw", + payload = payload, + ) + mesh.remember(EveMediaObservationDocument, observation.observationId, observation) + workers.execute { + runCatching { socket.sendBinary(EveMediaObservationDocument.encode(observation)) } + .onFailure { + sensorSocket?.close() + sensorSocket = null + connectSensorUplink() + } + } + } + private fun postBroker(text: String) = main.post { brokerText.text = text } private fun postSensor(text: String) = main.post { sensorText.text = text } private fun dp(value: Int): Int = (value * resources.displayMetrics.density + 0.5f).toInt()