diff --git a/Makefile b/Makefile index b98e69e..22de501 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,7 @@ EveCanvas_FILES = \ Sources/EVEViewController.mm \ Sources/EVEGLView.mm \ Sources/EVEFrameStreamClient.m \ + Sources/EVESensorUplinkClient.m \ Sources/EVEH264StreamDecoder.m EveCanvas_FRAMEWORKS = UIKit QuartzCore OpenGLES CoreMotion AVFoundation CoreMedia CoreVideo VideoToolbox diff --git a/README.md b/README.md index 6ce477c..e8b5920 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,12 @@ the development machine. - `CMMotionManager` for accelerometer and gyro telemetry. - `EVEFrameStreamClient` receives binary JPEG frames from the Starfire CEF relay and sends touch events back as JSON viewport coordinates. +- `EVESensorUplinkClient` opens separate WebSocket uplinks for camera and + microphone frame-events so sensor traffic does not block display/control + traffic. +- `EVEViewController` captures camera frames with AVFoundation and microphone + blocks with AVAudioEngine, then sends `eve-camera` and `eve-mic` samples to + Mimir. ## Invariants @@ -32,7 +38,8 @@ the development machine. - Starfire CEF owns browser pixels; EveCanvas owns display and touch capture. - UIKit is the current streamed-frame owner; OpenGL ES is fallback/local render. - The status bar stays hidden. -- Sensor reads are display-only telemetry until a later input model owns them. +- EveCanvas owns Eve-local sensor reads. Mimir owns synchronization and final + interpretation after those samples arrive. - CultMesh networking is not smuggled into the render shell yet. ## VoidBot CEF Stream @@ -46,6 +53,22 @@ npm run swarm:eve-cef-relay -- --width 1620 --height 2160 --scale 2 --port 8792 EveCanvas connects to `ws://192.168.1.66:8792/stream`, displays binary JPEG frames, and returns touch events to the relay. +## Mimir Sensor Uplink + +Start Mimir with `config/mimir-runtime.raven-eve.example.json` or run the +receiver processes directly: + +```powershell +dotnet run --project E:\Projects\Mimir\src\Mimir.EveSensorReceiver\Mimir.EveSensorReceiver.csproj -- --port 8793 --path /eve/camera --source-id eve-camera --type video-frame +dotnet run --project E:\Projects\Mimir\src\Mimir.EveSensorReceiver\Mimir.EveSensorReceiver.csproj -- --port 8794 --path /eve/mic --source-id eve-mic --type audio-block +``` + +EveCanvas sends camera frame-events to `ws://192.168.1.66:8793/eve/camera` and +microphone frame-events to `ws://192.168.1.66:8794/eve/mic`. The first transport +uses JSON plus base64 payloads because it is inspectable and already matches +Mimir's frame-event source. Replace it with binary framing only after the +sample contract is proven on device. + ## Build Shape The project is a Theos-style iOS application: @@ -101,4 +124,5 @@ without losing the basic app deployment path. - Confirm actual fullscreen drawable size on EVE. - Add a simple triangle/quad shader so the renderer proves more than clear. - Add touch/Pencil visual markers. -- Add a local telemetry bridge once the shell is stable. +- Replace JSON/base64 sensor packets with binary framing once camera and mic + timing are proven through Mimir. diff --git a/Resources/Info.plist b/Resources/Info.plist index 22ed019..befed8e 100644 --- a/Resources/Info.plist +++ b/Resources/Info.plist @@ -28,6 +28,10 @@ NSAllowsArbitraryLoads + NSCameraUsageDescription + EveCanvas sends camera frames to Mimir for synchronized scene capture. + NSMicrophoneUsageDescription + EveCanvas sends microphone blocks to Mimir for synchronized audio capture. UIApplicationSupportsIndirectInputEvents UIRequiresFullScreen diff --git a/Sources/EVESensorUplinkClient.h b/Sources/EVESensorUplinkClient.h new file mode 100644 index 0000000..ec1f863 --- /dev/null +++ b/Sources/EVESensorUplinkClient.h @@ -0,0 +1,12 @@ +#import + +@interface EVESensorUplinkClient : NSObject + +- (instancetype)initWithURLs:(NSArray *)urls label:(NSString *)label; +- (void)connect; +- (void)disconnect; +- (void)sendJSONObject:(NSDictionary *)payload; + +@property(nonatomic, copy, readonly) NSString *status; + +@end diff --git a/Sources/EVESensorUplinkClient.m b/Sources/EVESensorUplinkClient.m new file mode 100644 index 0000000..64aae12 --- /dev/null +++ b/Sources/EVESensorUplinkClient.m @@ -0,0 +1,110 @@ +#import "EVESensorUplinkClient.h" + +@interface EVESensorUplinkClient () + +@property(nonatomic, copy) NSArray *urls; +@property(nonatomic, copy) NSString *label; +@property(nonatomic, assign) NSUInteger urlIndex; +@property(nonatomic, strong) NSURLSession *session; +@property(nonatomic, strong) NSURLSessionWebSocketTask *task; +@property(nonatomic, assign) BOOL shouldReconnect; +@property(nonatomic, assign) BOOL connecting; +@property(nonatomic, copy, readwrite) NSString *status; + +@end + +@implementation EVESensorUplinkClient + +- (instancetype)initWithURLs:(NSArray *)urls label:(NSString *)label { + self = [super init]; + if (!self) { + return nil; + } + + self.urls = urls.count > 0 ? urls : @[]; + self.label = label ?: @"sensor"; + self.session = [NSURLSession sessionWithConfiguration:NSURLSessionConfiguration.defaultSessionConfiguration]; + self.status = @"idle"; + return self; +} + +- (void)connect { + self.shouldReconnect = YES; + if (self.connecting || self.task) { + return; + } + + NSURL *url = self.urls.count > 0 ? self.urls[self.urlIndex % self.urls.count] : nil; + if (!url) { + self.status = @"url missing"; + return; + } + + self.connecting = YES; + self.task = [self.session webSocketTaskWithURL:url]; + [self.task resume]; + self.connecting = NO; + self.status = [NSString stringWithFormat:@"%@ connecting %@", self.label, url.host ?: @""]; + NSLog(@"EveCanvas %@ uplink connecting %@", self.label, url.absoluteString); +} + +- (void)disconnect { + self.shouldReconnect = NO; + [self.task cancelWithCloseCode:NSURLSessionWebSocketCloseCodeNormalClosure reason:nil]; + self.task = nil; +} + +- (void)sendJSONObject:(NSDictionary *)payload { + NSURLSessionWebSocketTask *task = self.task; + if (!task) { + [self connect]; + return; + } + + NSError *jsonError = nil; + NSData *data = [NSJSONSerialization dataWithJSONObject:payload options:0 error:&jsonError]; + if (!data || jsonError) { + self.status = [NSString stringWithFormat:@"%@ json failed", self.label]; + return; + } + + NSString *text = [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding]; + NSURLSessionWebSocketMessage *message = [[NSURLSessionWebSocketMessage alloc] initWithString:text]; + __weak typeof(self) weakSelf = self; + [task sendMessage:message completionHandler:^(NSError *error) { + __strong typeof(weakSelf) self = weakSelf; + if (!self) { + return; + } + + if (error) { + self.status = [NSString stringWithFormat:@"%@ send failed", self.label]; + NSLog(@"EveCanvas %@ uplink send failed: %@", self.label, error); + self.task = nil; + [self reconnectSoon]; + } else { + self.status = [NSString stringWithFormat:@"%@ live", self.label]; + } + }]; +} + +- (void)reconnectSoon { + if (!self.shouldReconnect) { + return; + } + + if (self.urls.count > 1) { + self.urlIndex = (self.urlIndex + 1) % self.urls.count; + } + + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(2.0 * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{ + [self connect]; + }); +} + +- (void)dealloc { + [self disconnect]; + [self.session invalidateAndCancel]; +} + +@end diff --git a/Sources/EVEViewController.mm b/Sources/EVEViewController.mm index 67ddd7f..fe09d47 100644 --- a/Sources/EVEViewController.mm +++ b/Sources/EVEViewController.mm @@ -6,8 +6,16 @@ #import "EVEFrameStreamClient.h" #import "EVEGLView.h" #import "EVEH264StreamDecoder.h" +#import "EVESensorUplinkClient.h" -@interface EVEViewController () +@import AVFoundation; +@import CoreImage; + +static int64_t EVEHostTimeNowNs(void) { + return (int64_t)(CACurrentMediaTime() * 1000000000.0); +} + +@interface EVEViewController () @property(nonatomic, strong) EVEGLView *glView; @property(nonatomic, strong) UIImageView *streamImageView; @@ -16,9 +24,18 @@ @property(nonatomic, strong) CADisplayLink *displayLink; @property(nonatomic, strong) CMMotionManager *motionManager; @property(nonatomic, strong) EVEFrameStreamClient *streamClient; +@property(nonatomic, strong) EVESensorUplinkClient *cameraUplink; +@property(nonatomic, strong) EVESensorUplinkClient *micUplink; +@property(nonatomic, strong) AVCaptureSession *captureSession; +@property(nonatomic, strong) dispatch_queue_t captureQueue; +@property(nonatomic, strong) CIContext *ciContext; +@property(nonatomic, strong) AVAudioEngine *audioEngine; @property(nonatomic, assign) NSTimeInterval previousTimestamp; +@property(nonatomic, assign) NSTimeInterval previousCameraSendTimestamp; @property(nonatomic, assign) double filteredFPS; @property(nonatomic, assign) NSUInteger touchCount; +@property(nonatomic, assign) uint64_t cameraSequence; +@property(nonatomic, assign) uint64_t micSequence; @property(nonatomic, assign) CGSize streamViewportSize; @property(nonatomic, assign) CGFloat streamScale; @property(nonatomic, copy) NSString *streamStatus; @@ -93,6 +110,20 @@ self.streamClient = [[EVEFrameStreamClient alloc] initWithURLs:streamURLs delegate:self]; [self.streamClient connect]; + NSArray *cameraURLs = @[ + [NSURL URLWithString:@"ws://127.0.0.1:8793/eve/camera"], + [NSURL URLWithString:@"ws://192.168.1.66:8793/eve/camera"], + ]; + NSArray *micURLs = @[ + [NSURL URLWithString:@"ws://127.0.0.1:8794/eve/mic"], + [NSURL URLWithString:@"ws://192.168.1.66:8794/eve/mic"], + ]; + self.cameraUplink = [[EVESensorUplinkClient alloc] initWithURLs:cameraURLs label:@"camera"]; + self.micUplink = [[EVESensorUplinkClient alloc] initWithURLs:micURLs label:@"mic"]; + [self.cameraUplink connect]; + [self.micUplink connect]; + [self startSensorCapture]; + self.displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(frameTick:)]; [self.displayLink addToRunLoop:NSRunLoop.mainRunLoop forMode:NSRunLoopCommonModes]; } @@ -100,6 +131,10 @@ - (void)dealloc { [self.displayLink invalidate]; [self.streamClient disconnect]; + [self.cameraUplink disconnect]; + [self.micUplink disconnect]; + [self.captureSession stopRunning]; + [self.audioEngine stop]; [self.motionManager stopAccelerometerUpdates]; [self.motionManager stopGyroUpdates]; } @@ -153,6 +188,7 @@ "points %.0fx%.0f pixels %.0fx%.0f @ %.1fx\n" "stream %.0fx%.0f @ %.1fx\n" "fps %.1f touches %lu\n" + "uplink %@ / %@\n" "accel %+0.2f %+0.2f %+0.2f\n" "gyro %+0.2f %+0.2f %+0.2f", self.streamStatus ?: @"stream", @@ -160,11 +196,165 @@ points.width, points.height, pixels.width, pixels.height, scale, self.streamViewportSize.width, self.streamViewportSize.height, self.streamScale, self.filteredFPS, (unsigned long)self.touchCount, + self.cameraUplink.status ?: @"camera", + self.micUplink.status ?: @"mic", acceleration.x, acceleration.y, acceleration.z, rotation.x, rotation.y, rotation.z]; [self.overlayLabel sizeToFit]; } +- (void)startSensorCapture { + self.ciContext = [CIContext contextWithOptions:nil]; + self.captureQueue = dispatch_queue_create("org.gamecult.evecanvas.camera", DISPATCH_QUEUE_SERIAL); + + [AVCaptureDevice requestAccessForMediaType:AVMediaTypeVideo completionHandler:^(BOOL granted) { + NSLog(@"EveCanvas camera permission %@", granted ? @"granted" : @"denied"); + if (granted) { + [self startCameraCapture]; + } + }]; + + [AVCaptureDevice requestAccessForMediaType:AVMediaTypeAudio completionHandler:^(BOOL granted) { + NSLog(@"EveCanvas microphone permission %@", granted ? @"granted" : @"denied"); + if (granted) { + [self startMicrophoneCapture]; + } + }]; +} + +- (void)startCameraCapture { + NSLog(@"EveCanvas starting camera capture"); + AVCaptureSession *session = [[AVCaptureSession alloc] init]; + session.sessionPreset = AVCaptureSessionPreset640x480; + AVCaptureDevice *device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo]; + if (!device) { + NSLog(@"EveCanvas camera device missing"); + return; + } + + NSError *error = nil; + AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:device error:&error]; + if (!input || error || ![session canAddInput:input]) { + NSLog(@"EveCanvas camera input failed %@", error); + return; + } + [session addInput:input]; + + AVCaptureVideoDataOutput *output = [[AVCaptureVideoDataOutput alloc] init]; + output.alwaysDiscardsLateVideoFrames = YES; + output.videoSettings = @{ + (id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA), + }; + [output setSampleBufferDelegate:self queue:self.captureQueue]; + if (![session canAddOutput:output]) { + NSLog(@"EveCanvas camera output rejected"); + return; + } + [session addOutput:output]; + + self.captureSession = session; + [session startRunning]; + NSLog(@"EveCanvas camera capture running"); +} + +- (void)startMicrophoneCapture { + NSLog(@"EveCanvas starting microphone capture"); + AVAudioEngine *engine = [[AVAudioEngine alloc] init]; + AVAudioInputNode *input = engine.inputNode; + AVAudioFormat *format = [input outputFormatForBus:0]; + if (!format) { + NSLog(@"EveCanvas microphone format missing"); + return; + } + + __weak typeof(self) weakSelf = self; + [input installTapOnBus:0 bufferSize:1024 format:format block:^(AVAudioPCMBuffer *buffer, AVAudioTime *when) { + (void)when; + __strong typeof(weakSelf) self = weakSelf; + if (!self || buffer.frameLength == 0 || !buffer.floatChannelData) { + return; + } + + NSUInteger frameCount = buffer.frameLength; + NSUInteger channels = MAX((NSUInteger)1, MIN((NSUInteger)format.channelCount, (NSUInteger)2)); + NSMutableData *samples = [NSMutableData dataWithLength:frameCount * channels * sizeof(float)]; + float *dst = (float *)samples.mutableBytes; + for (NSUInteger frame = 0; frame < frameCount; frame++) { + for (NSUInteger channel = 0; channel < channels; channel++) { + dst[frame * channels + channel] = buffer.floatChannelData[channel][frame]; + } + } + + NSString *base64 = [samples base64EncodedStringWithOptions:0]; + NSDictionary *payload = @{ + @"type": @"audio-block", + @"sourceId": @"eve-mic", + @"timestampNs": @(EVEHostTimeNowNs()), + @"sequence": @(self.micSequence++), + @"sampleRate": @((int)format.sampleRate), + @"channels": @(channels), + @"sampleFormat": @"Float32", + @"frameCount": @(frameCount), + @"byteLength": @(samples.length), + @"samplesBase64": base64, + }; + [self.micUplink sendJSONObject:payload]; + }]; + + NSError *error = nil; + if ([engine startAndReturnError:&error]) { + self.audioEngine = engine; + NSLog(@"EveCanvas microphone capture running"); + } else { + NSLog(@"EveCanvas microphone start failed %@", error); + } +} + +- (void)captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection { + (void)output; + (void)connection; + + NSTimeInterval now = CACurrentMediaTime(); + if (self.previousCameraSendTimestamp > 0 && now - self.previousCameraSendTimestamp < 0.10) { + return; + } + self.previousCameraSendTimestamp = now; + + CVImageBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + if (!pixelBuffer) { + return; + } + + size_t width = CVPixelBufferGetWidth(pixelBuffer); + size_t height = CVPixelBufferGetHeight(pixelBuffer); + CIImage *image = [CIImage imageWithCVPixelBuffer:pixelBuffer]; + CGImageRef cgImage = [self.ciContext createCGImage:image fromRect:CGRectMake(0, 0, width, height)]; + if (!cgImage) { + return; + } + + UIImage *uiImage = [UIImage imageWithCGImage:cgImage]; + NSData *jpeg = UIImageJPEGRepresentation(uiImage, 0.55); + CGImageRelease(cgImage); + if (!jpeg) { + return; + } + + NSString *base64 = [jpeg base64EncodedStringWithOptions:0]; + NSDictionary *payload = @{ + @"type": @"video-frame", + @"sourceId": @"eve-camera", + @"timestampNs": @(EVEHostTimeNowNs()), + @"sequence": @(self.cameraSequence++), + @"width": @(width), + @"height": @(height), + @"pixelFormat": @"MJPG", + @"byteLength": @(jpeg.length), + @"samplesBase64": base64, + }; + [self.cameraUplink sendJSONObject:payload]; +} + - (CGPoint)streamPointForTouch:(UITouch *)touch { CGPoint viewPoint = [touch locationInView:self.streamImageView]; CGSize bounds = self.streamImageView.bounds.size;