#import "EVEViewController.h" #import #import #import "EVEFrameStreamClient.h" #import "EVEDashboardClient.h" #import "EVEGLView.h" #import "EVEH264StreamDecoder.h" #import "EVESensorUplinkClient.h" @import AVFoundation; static int64_t EVEHostTimeNowNs(void) { return (int64_t)(CACurrentMediaTime() * 1000000000.0); } @interface EVEViewController () @property(nonatomic, strong) EVEGLView *glView; @property(nonatomic, strong) UIImageView *streamImageView; @property(nonatomic, strong) EVEH264StreamDecoder *videoDecoder; @property(nonatomic, strong) UILabel *overlayLabel; @property(nonatomic, strong) UIView *dashboardView; @property(nonatomic, strong) UIView *sceneCanvasView; @property(nonatomic, strong) UIStackView *hierarchyStackView; @property(nonatomic, strong) UIStackView *toolbarStackView; @property(nonatomic, strong) UILabel *dashboardStatusLabel; @property(nonatomic, strong) NSMutableDictionary *nodeViews; @property(nonatomic, strong) NSMutableDictionary *dashboardNodes; @property(nonatomic, strong) CADisplayLink *displayLink; @property(nonatomic, strong) CMMotionManager *motionManager; @property(nonatomic, strong) EVEFrameStreamClient *streamClient; @property(nonatomic, strong) EVEDashboardClient *dashboardClient; @property(nonatomic, strong) EVESensorUplinkClient *cameraUplink; @property(nonatomic, strong) EVESensorUplinkClient *micUplink; @property(nonatomic, strong) AVCaptureSession *captureSession; @property(nonatomic, strong) dispatch_queue_t captureQueue; @property(nonatomic, strong) AVAudioEngine *audioEngine; @property(nonatomic, assign) NSTimeInterval previousTimestamp; @property(nonatomic, assign) NSTimeInterval previousCameraSendTimestamp; @property(nonatomic, assign) double filteredFPS; @property(nonatomic, assign) NSUInteger touchCount; @property(nonatomic, assign) uint64_t cameraSequence; @property(nonatomic, assign) uint64_t micSequence; @property(nonatomic, assign) CGSize streamViewportSize; @property(nonatomic, assign) CGFloat streamScale; @property(nonatomic, copy) NSString *streamStatus; @property(nonatomic, copy) NSString *streamCodec; @property(nonatomic, copy) NSString *dialogueLine; @property(nonatomic, copy) NSString *dashboardStatus; @property(nonatomic, copy) NSString *selectedNodeId; @property(nonatomic, assign) CGFloat activeGestureStartScale; @property(nonatomic, assign) CGFloat activeGestureStartRotation; @end @implementation EVEViewController - (void)loadView { UIView *root = [[UIView alloc] initWithFrame:UIScreen.mainScreen.bounds]; root.backgroundColor = UIColor.blackColor; root.multipleTouchEnabled = YES; self.glView = [[EVEGLView alloc] initWithFrame:root.bounds]; self.glView.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight; [root addSubview:self.glView]; self.streamImageView = [[UIImageView alloc] initWithFrame:root.bounds]; self.streamImageView.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight; self.streamImageView.contentMode = UIViewContentModeScaleAspectFit; self.streamImageView.backgroundColor = UIColor.blackColor; self.streamImageView.userInteractionEnabled = NO; [root addSubview:self.streamImageView]; self.videoDecoder = [[EVEH264StreamDecoder alloc] init]; self.videoDecoder.displayLayer.frame = root.bounds; self.videoDecoder.displayLayer.hidden = YES; [root.layer addSublayer:self.videoDecoder.displayLayer]; [self installDashboardInRoot:root]; self.overlayLabel = [[UILabel alloc] initWithFrame:CGRectZero]; self.overlayLabel.translatesAutoresizingMaskIntoConstraints = NO; self.overlayLabel.numberOfLines = 0; self.overlayLabel.textColor = UIColor.whiteColor; self.overlayLabel.backgroundColor = [[UIColor blackColor] colorWithAlphaComponent:0.45]; self.overlayLabel.font = [UIFont monospacedSystemFontOfSize:13.0 weight:UIFontWeightRegular]; self.overlayLabel.layer.cornerRadius = 6.0; self.overlayLabel.layer.masksToBounds = YES; [root addSubview:self.overlayLabel]; UILayoutGuide *safe = root.safeAreaLayoutGuide; [NSLayoutConstraint activateConstraints:@[ [self.overlayLabel.leadingAnchor constraintEqualToAnchor:safe.leadingAnchor constant:12.0], [self.overlayLabel.topAnchor constraintEqualToAnchor:safe.topAnchor constant:12.0], [self.overlayLabel.widthAnchor constraintLessThanOrEqualToAnchor:safe.widthAnchor multiplier:0.92], ]]; self.view = root; } - (void)viewDidLoad { [super viewDidLoad]; self.motionManager = [[CMMotionManager alloc] init]; self.motionManager.accelerometerUpdateInterval = 1.0 / 30.0; self.motionManager.gyroUpdateInterval = 1.0 / 30.0; if (self.motionManager.accelerometerAvailable) { [self.motionManager startAccelerometerUpdates]; } if (self.motionManager.gyroAvailable) { [self.motionManager startGyroUpdates]; } self.streamViewportSize = CGSizeMake(1620.0, 2160.0); self.streamScale = 2.0; self.streamStatus = @"stream idle"; self.streamCodec = @"jpeg"; self.dialogueLine = @"awaiting Mimir"; self.dashboardStatus = @"dashboard idle"; self.nodeViews = [NSMutableDictionary dictionary]; self.dashboardNodes = [NSMutableDictionary dictionary]; NSArray *streamURLs = @[ [NSURL URLWithString:@"ws://127.0.0.1:8792/stream"], [NSURL URLWithString:@"ws://192.168.1.66:8792/stream"], ]; self.streamClient = [[EVEFrameStreamClient alloc] initWithURLs:streamURLs delegate:self]; [self.streamClient connect]; NSArray *dashboardURLs = @[ [NSURL URLWithString:@"ws://127.0.0.1:8795/eve/deck"], [NSURL URLWithString:@"ws://192.168.1.66:8795/eve/deck"], [NSURL URLWithString:@"ws://127.0.0.1:8795/eve/dashboard"], [NSURL URLWithString:@"ws://192.168.1.66:8795/eve/dashboard"], ]; self.dashboardClient = [[EVEDashboardClient alloc] initWithURLs:dashboardURLs delegate:self]; [self.dashboardClient connect]; NSArray *cameraURLs = @[ [NSURL URLWithString:@"ws://127.0.0.1:8793/eve/camera"], [NSURL URLWithString:@"ws://192.168.1.66:8793/eve/camera"], ]; NSArray *micURLs = @[ [NSURL URLWithString:@"ws://127.0.0.1:8794/eve/mic"], [NSURL URLWithString:@"ws://192.168.1.66:8794/eve/mic"], ]; self.cameraUplink = [[EVESensorUplinkClient alloc] initWithURLs:cameraURLs label:@"camera"]; self.micUplink = [[EVESensorUplinkClient alloc] initWithURLs:micURLs label:@"mic"]; [self.cameraUplink connect]; [self.micUplink connect]; [self startSensorCapture]; self.displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(frameTick:)]; [self.displayLink addToRunLoop:NSRunLoop.mainRunLoop forMode:NSRunLoopCommonModes]; } - (void)dealloc { [self.displayLink invalidate]; [self.streamClient disconnect]; [self.dashboardClient disconnect]; [self.cameraUplink disconnect]; [self.micUplink disconnect]; [self.captureSession stopRunning]; [self.audioEngine stop]; [self.motionManager stopAccelerometerUpdates]; [self.motionManager stopGyroUpdates]; } - (BOOL)prefersStatusBarHidden { return YES; } - (UIRectEdge)preferredScreenEdgesDeferringSystemGestures { return UIRectEdgeAll; } - (BOOL)prefersHomeIndicatorAutoHidden { return YES; } - (void)viewDidLayoutSubviews { [super viewDidLayoutSubviews]; [self.glView resizeDrawableIfNeeded]; self.videoDecoder.displayLayer.frame = self.view.bounds; for (NSString *nodeId in self.dashboardNodes) { [self updateDashboardNodeView:self.dashboardNodes[nodeId]]; } } - (void)frameTick:(CADisplayLink *)link { if (self.previousTimestamp > 0) { NSTimeInterval delta = link.timestamp - self.previousTimestamp; if (delta > 0) { double instantFPS = 1.0 / delta; self.filteredFPS = self.filteredFPS == 0 ? instantFPS : (self.filteredFPS * 0.9 + instantFPS * 0.1); } } self.previousTimestamp = link.timestamp; [self.glView renderAtTime:link.timestamp]; [self updateOverlay]; } - (void)updateOverlay { if (self.dashboardView) { self.overlayLabel.hidden = YES; return; } self.overlayLabel.hidden = NO; UIScreen *screen = UIScreen.mainScreen; CGSize points = screen.bounds.size; CGFloat scale = screen.scale; CGSize pixels = CGSizeMake(points.width * scale, points.height * scale); CMAcceleration acceleration = self.motionManager.accelerometerData.acceleration; CMRotationRate rotation = self.motionManager.gyroData.rotationRate; self.overlayLabel.text = [NSString stringWithFormat: @"EVE Canvas\n" "CEF stream + native touch\n" "%@\n" "codec %@\n" "Mimir: %@\n" "dashboard %@\n" "points %.0fx%.0f pixels %.0fx%.0f @ %.1fx\n" "stream %.0fx%.0f @ %.1fx\n" "fps %.1f touches %lu\n" "uplink %@ / %@\n" "accel %+0.2f %+0.2f %+0.2f\n" "gyro %+0.2f %+0.2f %+0.2f", self.streamStatus ?: @"stream", self.streamCodec ?: @"unknown", self.dialogueLine ?: @"", self.dashboardStatus ?: @"dashboard", points.width, points.height, pixels.width, pixels.height, scale, self.streamViewportSize.width, self.streamViewportSize.height, self.streamScale, self.filteredFPS, (unsigned long)self.touchCount, self.cameraUplink.status ?: @"camera", self.micUplink.status ?: @"mic", acceleration.x, acceleration.y, acceleration.z, rotation.x, rotation.y, rotation.z]; [self.overlayLabel sizeToFit]; } - (void)installDashboardInRoot:(UIView *)root { self.dashboardView = [[UIView alloc] initWithFrame:root.bounds]; self.dashboardView.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight; self.dashboardView.backgroundColor = [[UIColor colorWithRed:0.02 green:0.025 blue:0.03 alpha:1.0] colorWithAlphaComponent:0.92]; [root addSubview:self.dashboardView]; self.sceneCanvasView = [[UIView alloc] initWithFrame:CGRectZero]; self.sceneCanvasView.translatesAutoresizingMaskIntoConstraints = NO; self.sceneCanvasView.backgroundColor = [UIColor colorWithRed:0.05 green:0.055 blue:0.06 alpha:1.0]; self.sceneCanvasView.layer.borderColor = [UIColor colorWithWhite:1.0 alpha:0.12].CGColor; self.sceneCanvasView.layer.borderWidth = 1.0; self.sceneCanvasView.multipleTouchEnabled = YES; [self.dashboardView addSubview:self.sceneCanvasView]; self.hierarchyStackView = [[UIStackView alloc] initWithFrame:CGRectZero]; self.hierarchyStackView.translatesAutoresizingMaskIntoConstraints = NO; self.hierarchyStackView.axis = UILayoutConstraintAxisVertical; self.hierarchyStackView.spacing = 6.0; self.hierarchyStackView.alignment = UIStackViewAlignmentFill; [self.dashboardView addSubview:self.hierarchyStackView]; self.toolbarStackView = [[UIStackView alloc] initWithFrame:CGRectZero]; self.toolbarStackView.translatesAutoresizingMaskIntoConstraints = NO; self.toolbarStackView.axis = UILayoutConstraintAxisHorizontal; self.toolbarStackView.spacing = 8.0; self.toolbarStackView.alignment = UIStackViewAlignmentCenter; [self.dashboardView addSubview:self.toolbarStackView]; self.dashboardStatusLabel = [[UILabel alloc] initWithFrame:CGRectZero]; self.dashboardStatusLabel.translatesAutoresizingMaskIntoConstraints = NO; self.dashboardStatusLabel.textColor = [UIColor colorWithWhite:0.90 alpha:1.0]; self.dashboardStatusLabel.font = [UIFont monospacedSystemFontOfSize:12.0 weight:UIFontWeightRegular]; self.dashboardStatusLabel.text = @"dashboard idle"; [self.dashboardView addSubview:self.dashboardStatusLabel]; UILayoutGuide *safe = self.dashboardView.safeAreaLayoutGuide; [NSLayoutConstraint activateConstraints:@[ [self.hierarchyStackView.leadingAnchor constraintEqualToAnchor:safe.leadingAnchor constant:14.0], [self.hierarchyStackView.topAnchor constraintEqualToAnchor:safe.topAnchor constant:74.0], [self.hierarchyStackView.widthAnchor constraintEqualToConstant:230.0], [self.hierarchyStackView.bottomAnchor constraintLessThanOrEqualToAnchor:safe.bottomAnchor constant:-18.0], [self.sceneCanvasView.leadingAnchor constraintEqualToAnchor:self.hierarchyStackView.trailingAnchor constant:14.0], [self.sceneCanvasView.topAnchor constraintEqualToAnchor:safe.topAnchor constant:74.0], [self.sceneCanvasView.trailingAnchor constraintEqualToAnchor:safe.trailingAnchor constant:-14.0], [self.sceneCanvasView.bottomAnchor constraintEqualToAnchor:safe.bottomAnchor constant:-64.0], [self.toolbarStackView.leadingAnchor constraintEqualToAnchor:self.sceneCanvasView.leadingAnchor], [self.toolbarStackView.topAnchor constraintEqualToAnchor:safe.topAnchor constant:18.0], [self.toolbarStackView.trailingAnchor constraintLessThanOrEqualToAnchor:safe.trailingAnchor constant:-14.0], [self.toolbarStackView.heightAnchor constraintEqualToConstant:42.0], [self.dashboardStatusLabel.leadingAnchor constraintEqualToAnchor:self.sceneCanvasView.leadingAnchor], [self.dashboardStatusLabel.topAnchor constraintEqualToAnchor:self.sceneCanvasView.bottomAnchor constant:10.0], [self.dashboardStatusLabel.trailingAnchor constraintEqualToAnchor:self.sceneCanvasView.trailingAnchor], ]]; NSArray *buttons = @[ @{@"title": @"Reset", @"action": @"reset-transform"}, @{@"title": @"Hide/Show", @"action": @"toggle-visibility"}, ]; for (NSDictionary *entry in buttons) { UIButton *button = [UIButton buttonWithType:UIButtonTypeSystem]; [button setTitle:entry[@"title"] forState:UIControlStateNormal]; button.accessibilityIdentifier = entry[@"action"]; button.titleLabel.font = [UIFont systemFontOfSize:15.0 weight:UIFontWeightSemibold]; button.tintColor = UIColor.whiteColor; button.backgroundColor = [UIColor colorWithRed:0.16 green:0.18 blue:0.20 alpha:1.0]; button.layer.cornerRadius = 6.0; button.contentEdgeInsets = UIEdgeInsetsMake(8, 12, 8, 12); [button addTarget:self action:@selector(toolbarButtonPressed:) forControlEvents:UIControlEventTouchUpInside]; [self.toolbarStackView addArrangedSubview:button]; } } - (void)startSensorCapture { self.captureQueue = dispatch_queue_create("org.gamecult.evecanvas.camera", DISPATCH_QUEUE_SERIAL); [AVCaptureDevice requestAccessForMediaType:AVMediaTypeVideo completionHandler:^(BOOL granted) { NSLog(@"EveCanvas camera permission %@", granted ? @"granted" : @"denied"); if (granted) { [self startCameraCapture]; } }]; [AVCaptureDevice requestAccessForMediaType:AVMediaTypeAudio completionHandler:^(BOOL granted) { NSLog(@"EveCanvas microphone permission %@", granted ? @"granted" : @"denied"); if (granted) { [self startMicrophoneCapture]; } }]; } - (void)startCameraCapture { NSLog(@"EveCanvas starting camera capture"); AVCaptureSession *session = [[AVCaptureSession alloc] init]; session.sessionPreset = AVCaptureSessionPreset640x480; AVCaptureDevice *device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo]; if (!device) { NSLog(@"EveCanvas camera device missing"); return; } NSError *error = nil; AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:device error:&error]; if (!input || error || ![session canAddInput:input]) { NSLog(@"EveCanvas camera input failed %@", error); return; } [session addInput:input]; AVCaptureVideoDataOutput *output = [[AVCaptureVideoDataOutput alloc] init]; output.alwaysDiscardsLateVideoFrames = YES; output.videoSettings = @{ (id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA), }; [output setSampleBufferDelegate:self queue:self.captureQueue]; if (![session canAddOutput:output]) { NSLog(@"EveCanvas camera output rejected"); return; } [session addOutput:output]; self.captureSession = session; [session startRunning]; NSLog(@"EveCanvas camera capture running"); } - (void)startMicrophoneCapture { NSLog(@"EveCanvas starting microphone capture"); AVAudioEngine *engine = [[AVAudioEngine alloc] init]; AVAudioInputNode *input = engine.inputNode; AVAudioFormat *format = [input outputFormatForBus:0]; if (!format) { NSLog(@"EveCanvas microphone format missing"); return; } __weak typeof(self) weakSelf = self; [input installTapOnBus:0 bufferSize:1024 format:format block:^(AVAudioPCMBuffer *buffer, AVAudioTime *when) { (void)when; __strong typeof(weakSelf) self = weakSelf; if (!self || buffer.frameLength == 0 || !buffer.floatChannelData) { return; } NSUInteger frameCount = buffer.frameLength; NSUInteger channels = MAX((NSUInteger)1, MIN((NSUInteger)format.channelCount, (NSUInteger)2)); NSMutableData *samples = [NSMutableData dataWithLength:frameCount * channels * sizeof(float)]; float *dst = (float *)samples.mutableBytes; for (NSUInteger frame = 0; frame < frameCount; frame++) { for (NSUInteger channel = 0; channel < channels; channel++) { dst[frame * channels + channel] = buffer.floatChannelData[channel][frame]; } } NSString *base64 = [samples base64EncodedStringWithOptions:0]; NSDictionary *payload = @{ @"type": @"audio-block", @"sourceId": @"eve-mic", @"timestampNs": @(EVEHostTimeNowNs()), @"sequence": @(self.micSequence++), @"sampleRate": @((int)format.sampleRate), @"channels": @(channels), @"sampleFormat": @"Float32", @"frameCount": @(frameCount), @"byteLength": @(samples.length), @"samplesBase64": base64, }; [self.micUplink sendJSONObject:payload]; }]; NSError *error = nil; if ([engine startAndReturnError:&error]) { self.audioEngine = engine; NSLog(@"EveCanvas microphone capture running"); } else { NSLog(@"EveCanvas microphone start failed %@", error); } } - (void)captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection { (void)output; (void)connection; NSTimeInterval now = CACurrentMediaTime(); if (self.previousCameraSendTimestamp > 0 && now - self.previousCameraSendTimestamp < 0.10) { return; } self.previousCameraSendTimestamp = now; CVImageBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); if (!pixelBuffer) { return; } CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); size_t width = CVPixelBufferGetWidth(pixelBuffer); size_t height = CVPixelBufferGetHeight(pixelBuffer); size_t bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer); void *baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer); CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); CGContextRef bitmapContext = CGBitmapContextCreate(baseAddress, width, height, 8, bytesPerRow, colorSpace, kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst); CGImageRef cgImage = bitmapContext ? CGBitmapContextCreateImage(bitmapContext) : NULL; if (bitmapContext) { CGContextRelease(bitmapContext); } CGColorSpaceRelease(colorSpace); if (!cgImage) { CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); return; } UIImage *uiImage = [UIImage imageWithCGImage:cgImage]; NSData *jpeg = UIImageJPEGRepresentation(uiImage, 0.55); CGImageRelease(cgImage); CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); if (!jpeg) { return; } NSString *base64 = [jpeg base64EncodedStringWithOptions:0]; NSDictionary *payload = @{ @"type": @"video-frame", @"sourceId": @"eve-camera", @"timestampNs": @(EVEHostTimeNowNs()), @"sequence": @(self.cameraSequence++), @"width": @(width), @"height": @(height), @"pixelFormat": @"MJPG", @"byteLength": @(jpeg.length), @"samplesBase64": base64, }; [self.cameraUplink sendJSONObject:payload]; } - (CGPoint)streamPointForTouch:(UITouch *)touch { CGPoint viewPoint = [touch locationInView:self.streamImageView]; CGSize bounds = self.streamImageView.bounds.size; CGSize viewport = self.streamViewportSize; if (viewport.width <= 0 || viewport.height <= 0 || bounds.width <= 0 || bounds.height <= 0) { return CGPointZero; } CGFloat imageAspect = viewport.width / viewport.height; CGFloat boundsAspect = bounds.width / bounds.height; CGFloat drawWidth = bounds.width; CGFloat drawHeight = bounds.height; CGFloat offsetX = 0.0; CGFloat offsetY = 0.0; if (boundsAspect > imageAspect) { drawWidth = bounds.height * imageAspect; offsetX = (bounds.width - drawWidth) * 0.5; } else { drawHeight = bounds.width / imageAspect; offsetY = (bounds.height - drawHeight) * 0.5; } CGFloat x = (viewPoint.x - offsetX) * viewport.width / drawWidth; CGFloat y = (viewPoint.y - offsetY) * viewport.height / drawHeight; x = MIN(MAX(x, 0.0), viewport.width - 1.0); y = MIN(MAX(y, 0.0), viewport.height - 1.0); return CGPointMake(x, y); } - (void)dashboardClient:(EVEDashboardClient *)client didReceiveState:(NSDictionary *)state { (void)client; NSString *title = [state[@"title"] isKindOfClass:NSString.class] ? state[@"title"] : @"dashboard"; self.dashboardStatus = [NSString stringWithFormat:@"%@ v%@", title, state[@"version"] ?: @"?"]; self.dashboardStatusLabel.text = self.dashboardStatus; NSString *selected = state[@"selectedNodeId"]; if ([selected isKindOfClass:NSString.class]) { self.selectedNodeId = selected; } NSArray *nodes = state[@"nodes"]; if (![nodes isKindOfClass:NSArray.class]) { return; } [self.dashboardNodes removeAllObjects]; for (NSDictionary *node in nodes) { if (![node isKindOfClass:NSDictionary.class]) { continue; } NSString *nodeId = node[@"id"]; if (![nodeId isKindOfClass:NSString.class]) { continue; } self.dashboardNodes[nodeId] = node; [self ensureDashboardNodeView:node]; [self updateDashboardNodeView:node]; } [self rebuildHierarchyWithNodes:nodes]; } - (void)dashboardClient:(EVEDashboardClient *)client didChangeStatus:(NSString *)status { (void)client; self.dashboardStatus = status; self.dashboardStatusLabel.text = status; } - (void)ensureDashboardNodeView:(NSDictionary *)node { NSString *nodeId = node[@"id"]; if (self.nodeViews[nodeId]) { return; } UIView *view = [[UIView alloc] initWithFrame:CGRectMake(0, 0, 120, 80)]; view.backgroundColor = [UIColor colorWithRed:0.10 green:0.13 blue:0.16 alpha:1.0]; view.layer.borderWidth = 1.0; view.layer.cornerRadius = 6.0; view.multipleTouchEnabled = YES; view.accessibilityIdentifier = nodeId; UILabel *label = [[UILabel alloc] initWithFrame:CGRectInset(view.bounds, 8.0, 6.0)]; label.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight; label.numberOfLines = 3; label.textColor = UIColor.whiteColor; label.font = [UIFont monospacedSystemFontOfSize:12.0 weight:UIFontWeightMedium]; label.tag = 1001; [view addSubview:label]; UITapGestureRecognizer *tap = [[UITapGestureRecognizer alloc] initWithTarget:self action:@selector(nodeTapped:)]; [view addGestureRecognizer:tap]; UIPanGestureRecognizer *pan = [[UIPanGestureRecognizer alloc] initWithTarget:self action:@selector(nodePanned:)]; [view addGestureRecognizer:pan]; UIPinchGestureRecognizer *pinch = [[UIPinchGestureRecognizer alloc] initWithTarget:self action:@selector(nodePinched:)]; pinch.delegate = self; [view addGestureRecognizer:pinch]; UIRotationGestureRecognizer *rotation = [[UIRotationGestureRecognizer alloc] initWithTarget:self action:@selector(nodeRotated:)]; rotation.delegate = self; [view addGestureRecognizer:rotation]; [self.sceneCanvasView addSubview:view]; self.nodeViews[nodeId] = view; } - (void)updateDashboardNodeView:(NSDictionary *)node { NSString *nodeId = node[@"id"]; UIView *view = self.nodeViews[nodeId]; if (!view) { return; } CGFloat canvasWidth = MAX(1.0, self.sceneCanvasView.bounds.size.width); CGFloat canvasHeight = MAX(1.0, self.sceneCanvasView.bounds.size.height); CGFloat width = MAX(72.0, [node[@"width"] doubleValue] * canvasWidth); CGFloat height = MAX(54.0, [node[@"height"] doubleValue] * canvasHeight); CGFloat x = ([node[@"x"] doubleValue] + 1.0) * 0.5 * canvasWidth; CGFloat y = ([node[@"y"] doubleValue] + 1.0) * 0.5 * canvasHeight; view.bounds = CGRectMake(0, 0, width, height); view.center = CGPointMake(x, y); CGFloat scale = MAX(0.25, [node[@"scale"] doubleValue]); CGFloat rotation = [node[@"rotation"] doubleValue]; view.transform = CGAffineTransformRotate(CGAffineTransformMakeScale(scale, scale), rotation); BOOL visible = [node[@"visible"] boolValue]; view.alpha = visible ? 1.0 : 0.28; BOOL selected = [nodeId isEqualToString:self.selectedNodeId]; view.layer.borderColor = selected ? [UIColor colorWithRed:0.45 green:0.80 blue:1.0 alpha:1.0].CGColor : [UIColor colorWithWhite:1.0 alpha:0.16].CGColor; view.layer.borderWidth = selected ? 2.0 : 1.0; UILabel *label = (UILabel *)[view viewWithTag:1001]; if ([label isKindOfClass:UILabel.class]) { label.text = [NSString stringWithFormat:@"%@\n%@ %@", node[@"label"] ?: nodeId, node[@"kind"] ?: @"source", node[@"health"] ?: @""]; } } - (void)rebuildHierarchyWithNodes:(NSArray *)nodes { for (UIView *view in self.hierarchyStackView.arrangedSubviews) { [self.hierarchyStackView removeArrangedSubview:view]; [view removeFromSuperview]; } UILabel *title = [[UILabel alloc] initWithFrame:CGRectZero]; title.text = @"Dashboards"; title.textColor = UIColor.whiteColor; title.font = [UIFont systemFontOfSize:16.0 weight:UIFontWeightBold]; [self.hierarchyStackView addArrangedSubview:title]; for (NSDictionary *node in nodes) { NSString *nodeId = node[@"id"]; if (![nodeId isKindOfClass:NSString.class]) { continue; } UIButton *button = [UIButton buttonWithType:UIButtonTypeSystem]; NSString *visible = [node[@"visible"] boolValue] ? @"●" : @"○"; [button setTitle:[NSString stringWithFormat:@"%@ %@", visible, node[@"label"] ?: nodeId] forState:UIControlStateNormal]; button.contentHorizontalAlignment = UIControlContentHorizontalAlignmentLeft; button.accessibilityIdentifier = nodeId; button.titleLabel.font = [UIFont monospacedSystemFontOfSize:13.0 weight:[nodeId isEqualToString:self.selectedNodeId] ? UIFontWeightBold : UIFontWeightRegular]; button.tintColor = [nodeId isEqualToString:self.selectedNodeId] ? [UIColor colorWithRed:0.55 green:0.85 blue:1.0 alpha:1.0] : [UIColor colorWithWhite:0.85 alpha:1.0]; [button addTarget:self action:@selector(hierarchyNodePressed:) forControlEvents:UIControlEventTouchUpInside]; [self.hierarchyStackView addArrangedSubview:button]; } } - (void)nodeTapped:(UITapGestureRecognizer *)recognizer { NSString *nodeId = recognizer.view.accessibilityIdentifier; if (!nodeId) { return; } [self selectOrOpenDashboardNode:nodeId]; } - (void)nodePanned:(UIPanGestureRecognizer *)recognizer { UIView *view = recognizer.view; NSString *nodeId = view.accessibilityIdentifier; if (!nodeId) { return; } CGPoint translation = [recognizer translationInView:self.sceneCanvasView]; view.center = CGPointMake(view.center.x + translation.x, view.center.y + translation.y); [recognizer setTranslation:CGPointZero inView:self.sceneCanvasView]; if (recognizer.state == UIGestureRecognizerStateChanged || recognizer.state == UIGestureRecognizerStateEnded) { [self sendMoveForNodeId:nodeId center:view.center]; } } - (void)nodePinched:(UIPinchGestureRecognizer *)recognizer { NSString *nodeId = recognizer.view.accessibilityIdentifier; NSDictionary *node = nodeId ? self.dashboardNodes[nodeId] : nil; if (!nodeId || !node) { return; } if (recognizer.state == UIGestureRecognizerStateBegan) { self.activeGestureStartScale = MAX(0.25, [node[@"scale"] doubleValue]); } CGFloat scale = MAX(0.25, MIN(3.0, self.activeGestureStartScale * recognizer.scale)); [self.dashboardClient sendCommand:@{@"type": @"scale", @"nodeId": nodeId, @"scale": @(scale)}]; } - (void)nodeRotated:(UIRotationGestureRecognizer *)recognizer { NSString *nodeId = recognizer.view.accessibilityIdentifier; NSDictionary *node = nodeId ? self.dashboardNodes[nodeId] : nil; if (!nodeId || !node) { return; } if (recognizer.state == UIGestureRecognizerStateBegan) { self.activeGestureStartRotation = [node[@"rotation"] doubleValue]; } CGFloat rotation = self.activeGestureStartRotation + recognizer.rotation; [self.dashboardClient sendCommand:@{@"type": @"rotate", @"nodeId": nodeId, @"rotation": @(rotation)}]; } - (void)sendMoveForNodeId:(NSString *)nodeId center:(CGPoint)center { CGFloat canvasWidth = MAX(1.0, self.sceneCanvasView.bounds.size.width); CGFloat canvasHeight = MAX(1.0, self.sceneCanvasView.bounds.size.height); CGFloat x = MIN(1.0, MAX(-1.0, (center.x / canvasWidth) * 2.0 - 1.0)); CGFloat y = MIN(1.0, MAX(-1.0, (center.y / canvasHeight) * 2.0 - 1.0)); [self.dashboardClient sendCommand:@{@"type": @"move", @"nodeId": nodeId, @"x": @(x), @"y": @(y)}]; } - (void)hierarchyNodePressed:(UIButton *)button { NSString *nodeId = button.accessibilityIdentifier; if (!nodeId) { return; } [self selectOrOpenDashboardNode:nodeId]; } - (void)selectOrOpenDashboardNode:(NSString *)nodeId { self.selectedNodeId = nodeId; NSDictionary *node = self.dashboardNodes[nodeId]; NSString *command = [node[@"command"] isKindOfClass:NSString.class] ? node[@"command"] : nil; NSString *providerId = [node[@"providerId"] isKindOfClass:NSString.class] ? node[@"providerId"] : nil; if ([command isEqualToString:@"open-provider"] && providerId) { [self.dashboardClient sendCommand:@{@"type": @"open-provider", @"nodeId": nodeId, @"providerId": providerId}]; return; } [self.dashboardClient sendCommand:@{@"type": @"select", @"nodeId": nodeId}]; } - (void)toolbarButtonPressed:(UIButton *)button { NSString *nodeId = self.selectedNodeId; NSString *action = button.accessibilityIdentifier; if (!nodeId || !action) { return; } [self.dashboardClient sendCommand:@{@"type": action, @"nodeId": nodeId}]; } - (BOOL)gestureRecognizer:(UIGestureRecognizer *)gestureRecognizer shouldRecognizeSimultaneouslyWithGestureRecognizer:(UIGestureRecognizer *)otherGestureRecognizer { (void)gestureRecognizer; (void)otherGestureRecognizer; return YES; } - (void)sendTouches:(NSSet *)touches event:(UIEvent *)event phase:(NSString *)phase { (void)event; UITouch *touch = touches.anyObject; if (!touch) { return; } CGPoint point = [self streamPointForTouch:touch]; [self.streamClient sendPointerPhase:phase x:point.x y:point.y]; } - (void)touchesBegan:(NSSet *)touches withEvent:(UIEvent *)event { self.touchCount = event.allTouches.count; [self sendTouches:touches event:event phase:@"down"]; } - (void)touchesMoved:(NSSet *)touches withEvent:(UIEvent *)event { self.touchCount = event.allTouches.count; [self sendTouches:touches event:event phase:@"move"]; } - (void)touchesEnded:(NSSet *)touches withEvent:(UIEvent *)event { self.touchCount = event.allTouches.count; [self sendTouches:touches event:event phase:@"up"]; } - (void)touchesCancelled:(NSSet *)touches withEvent:(UIEvent *)event { self.touchCount = event.allTouches.count; [self sendTouches:touches event:event phase:@"up"]; } - (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveImage:(UIImage *)image { (void)client; self.videoDecoder.displayLayer.hidden = YES; self.streamImageView.hidden = NO; self.streamImageView.image = image; } - (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveVideoAccessUnit:(NSData *)data { (void)client; self.streamImageView.hidden = YES; self.videoDecoder.displayLayer.hidden = NO; [self.videoDecoder consumeAnnexBAccessUnit:data]; } - (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveViewportWidth:(CGFloat)width height:(CGFloat)height scale:(CGFloat)scale { (void)client; if (width > 0 && height > 0) { self.streamViewportSize = CGSizeMake(width, height); } if (scale > 0) { self.streamScale = scale; } } - (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveCodec:(NSString *)codec { (void)client; if (![codec isEqualToString:self.streamCodec]) { [self.videoDecoder reset]; } self.streamCodec = codec; } - (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveDialogueText:(NSString *)text { (void)client; self.dialogueLine = text; } - (void)frameStreamClient:(EVEFrameStreamClient *)client didChangeStatus:(NSString *)status { (void)client; self.streamStatus = status; } @end