diff --git a/Makefile b/Makefile index 974cc18..38612ac 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,8 @@ EveCanvas_FILES = \ Sources/main.m \ Sources/EVEAppDelegate.m \ Sources/EVEViewController.mm \ - Sources/EVEGLView.mm + Sources/EVEGLView.mm \ + Sources/EVEFrameStreamClient.m EveCanvas_FRAMEWORKS = UIKit QuartzCore OpenGLES CoreMotion EveCanvas_CFLAGS = -fobjc-arc -Wall -Wextra -DGLES_SILENCE_DEPRECATION diff --git a/README.md b/README.md index 5090745..52b9b1b 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,10 @@ Fullscreen native render shell for EVE, the jailbroken iPad 8th generation test device. -This is deliberately not a web app. UIKit owns the app/window lifecycle and the -text overlay. OpenGL ES 3.0 owns the render surface. +This is deliberately not a web app. UIKit owns the app/window lifecycle, the +streamed frame surface, touch capture, and the text overlay. OpenGL ES remains a +local render fallback, but the VoidBot dashboard path is a native frame stream +from Starfire. ## Objective @@ -15,21 +17,35 @@ the development machine. - `EVEAppDelegate` creates one fullscreen `UIWindow`. - `EVEViewController` installs: - - `EVEGLView`, a `CAEAGLLayer` backed OpenGL ES 3.0 render target. - - a native UIKit `UILabel` overlay for crisp Retina text. + - `EVEGLView`, a `CAEAGLLayer` backed OpenGL ES render target. + - a full-screen `UIImageView` that displays CEF relay frames. + - `EVEFrameStreamClient`, a native WebSocket client for frame/input transport. + - a native UIKit `UILabel` overlay for crisp Retina status text. - `CADisplayLink` for frame ticking. - `CMMotionManager` for accelerometer and gyro telemetry. -- `EVEGLView` clears the full drawable with a slow color pulse and presents the - renderbuffer every frame. +- `EVEFrameStreamClient` receives binary JPEG frames from the Starfire CEF relay + and sends touch events back as JSON viewport coordinates. ## Invariants -- Browser/PWA layout is not part of this path. -- UIKit is the text owner; OpenGL ES is the pixel owner. +- Browser/PWA layout is not part of the iPad path. +- Starfire CEF owns browser pixels; EveCanvas owns display and touch capture. +- UIKit is the current streamed-frame owner; OpenGL ES is fallback/local render. - The status bar stays hidden. - Sensor reads are display-only telemetry until a later input model owns them. - CultMesh networking is not smuggled into the render shell yet. +## VoidBot CEF Stream + +Start the Starfire relay from `E:\Projects\VoidBot`: + +```powershell +npm run swarm:eve-cef-relay -- --width 2160 --height 1620 --scale 2 --port 8791 +``` + +EveCanvas connects to `ws://192.168.1.66:8791/stream`, displays binary JPEG +frames, and returns touch events to the relay. + ## Build Shape The project is a Theos-style iOS application: diff --git a/Resources/Info.plist b/Resources/Info.plist index 3a3f12e..22ed019 100644 --- a/Resources/Info.plist +++ b/Resources/Info.plist @@ -23,6 +23,11 @@ 1 LSRequiresIPhoneOS + NSAppTransportSecurity + + NSAllowsArbitraryLoads + + UIApplicationSupportsIndirectInputEvents UIRequiresFullScreen diff --git a/Sources/EVEFrameStreamClient.h b/Sources/EVEFrameStreamClient.h new file mode 100644 index 0000000..16f8fa8 --- /dev/null +++ b/Sources/EVEFrameStreamClient.h @@ -0,0 +1,20 @@ +#import + +@protocol EVEFrameStreamClientDelegate; + +@interface EVEFrameStreamClient : NSObject + +- (instancetype)initWithURL:(NSURL *)url delegate:(id)delegate; +- (void)connect; +- (void)disconnect; +- (void)sendPointerPhase:(NSString *)phase x:(CGFloat)x y:(CGFloat)y; + +@end + +@protocol EVEFrameStreamClientDelegate + +- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveImage:(UIImage *)image; +- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveViewportWidth:(CGFloat)width height:(CGFloat)height scale:(CGFloat)scale; +- (void)frameStreamClient:(EVEFrameStreamClient *)client didChangeStatus:(NSString *)status; + +@end diff --git a/Sources/EVEFrameStreamClient.m b/Sources/EVEFrameStreamClient.m new file mode 100644 index 0000000..4799ecf --- /dev/null +++ b/Sources/EVEFrameStreamClient.m @@ -0,0 +1,145 @@ +#import "EVEFrameStreamClient.h" + +@interface EVEFrameStreamClient () + +@property(nonatomic, strong) NSURL *url; +@property(nonatomic, weak) id delegate; +@property(nonatomic, strong) NSURLSession *session; +@property(nonatomic, strong) NSURLSessionWebSocketTask *task; +@property(nonatomic, assign) BOOL shouldReconnect; + +@end + +@implementation EVEFrameStreamClient + +- (instancetype)initWithURL:(NSURL *)url delegate:(id)delegate { + self = [super init]; + if (!self) { + return nil; + } + + self.url = url; + self.delegate = delegate; + self.session = [NSURLSession sessionWithConfiguration:NSURLSessionConfiguration.defaultSessionConfiguration]; + self.shouldReconnect = YES; + return self; +} + +- (void)connect { + self.shouldReconnect = YES; + [self.task cancelWithCloseCode:NSURLSessionWebSocketCloseCodeNormalClosure reason:nil]; + self.task = [self.session webSocketTaskWithURL:self.url]; + [self.task resume]; + [self publishStatus:@"connecting"]; + [self receiveNextMessage]; +} + +- (void)disconnect { + self.shouldReconnect = NO; + [self.task cancelWithCloseCode:NSURLSessionWebSocketCloseCodeNormalClosure reason:nil]; + self.task = nil; +} + +- (void)sendPointerPhase:(NSString *)phase x:(CGFloat)x y:(CGFloat)y { + if (!self.task) { + return; + } + + NSDictionary *payload = @{ + @"type": @"pointer", + @"phase": phase, + @"x": @(x), + @"y": @(y), + }; + NSData *json = [NSJSONSerialization dataWithJSONObject:payload options:0 error:nil]; + if (!json) { + return; + } + + NSString *text = [[NSString alloc] initWithData:json encoding:NSUTF8StringEncoding]; + NSURLSessionWebSocketMessage *message = [[NSURLSessionWebSocketMessage alloc] initWithString:text]; + [self.task sendMessage:message completionHandler:^(NSError *error) { + if (error) { + [self publishStatus:@"input send failed"]; + } + }]; +} + +- (void)receiveNextMessage { + __weak typeof(self) weakSelf = self; + [self.task receiveMessageWithCompletionHandler:^(NSURLSessionWebSocketMessage *message, NSError *error) { + __strong typeof(weakSelf) self = weakSelf; + if (!self) { + return; + } + + if (error) { + [self publishStatus:@"stream disconnected"]; + [self reconnectSoon]; + return; + } + + if (message.type == NSURLSessionWebSocketMessageTypeData) { + UIImage *image = [UIImage imageWithData:message.data]; + if (image) { + dispatch_async(dispatch_get_main_queue(), ^{ + [self.delegate frameStreamClient:self didReceiveImage:image]; + }); + } + } else if (message.type == NSURLSessionWebSocketMessageTypeString) { + [self handleTextMessage:message.string]; + } + + [self receiveNextMessage]; + }]; +} + +- (void)handleTextMessage:(NSString *)text { + NSData *data = [text dataUsingEncoding:NSUTF8StringEncoding]; + if (!data) { + return; + } + + NSDictionary *message = [NSJSONSerialization JSONObjectWithData:data options:0 error:nil]; + if (![message isKindOfClass:NSDictionary.class]) { + return; + } + + if ([message[@"type"] isEqualToString:@"config"]) { + CGFloat width = [message[@"width"] doubleValue]; + CGFloat height = [message[@"height"] doubleValue]; + CGFloat scale = [message[@"DeviceScaleFactor"] doubleValue]; + if (scale <= 0) { + scale = [message[@"deviceScaleFactor"] doubleValue]; + } + dispatch_async(dispatch_get_main_queue(), ^{ + [self.delegate frameStreamClient:self didReceiveViewportWidth:width height:height scale:scale]; + [self.delegate frameStreamClient:self didChangeStatus:@"stream live"]; + }); + } +} + +- (void)reconnectSoon { + if (!self.shouldReconnect) { + return; + } + + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(2.0 * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{ + if (self.shouldReconnect) { + [self connect]; + } + }); +} + +- (void)publishStatus:(NSString *)status { + dispatch_async(dispatch_get_main_queue(), ^{ + [self.delegate frameStreamClient:self didChangeStatus:status]; + }); +} + +- (void)dealloc { + [self disconnect]; + [self.session invalidateAndCancel]; +} + +@end diff --git a/Sources/EVEViewController.mm b/Sources/EVEViewController.mm index 60e8270..128b285 100644 --- a/Sources/EVEViewController.mm +++ b/Sources/EVEViewController.mm @@ -3,17 +3,23 @@ #import #import +#import "EVEFrameStreamClient.h" #import "EVEGLView.h" -@interface EVEViewController () +@interface EVEViewController () @property(nonatomic, strong) EVEGLView *glView; +@property(nonatomic, strong) UIImageView *streamImageView; @property(nonatomic, strong) UILabel *overlayLabel; @property(nonatomic, strong) CADisplayLink *displayLink; @property(nonatomic, strong) CMMotionManager *motionManager; +@property(nonatomic, strong) EVEFrameStreamClient *streamClient; @property(nonatomic, assign) NSTimeInterval previousTimestamp; @property(nonatomic, assign) double filteredFPS; @property(nonatomic, assign) NSUInteger touchCount; +@property(nonatomic, assign) CGSize streamViewportSize; +@property(nonatomic, assign) CGFloat streamScale; +@property(nonatomic, copy) NSString *streamStatus; @end @@ -28,6 +34,13 @@ self.glView.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight; [root addSubview:self.glView]; + self.streamImageView = [[UIImageView alloc] initWithFrame:root.bounds]; + self.streamImageView.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight; + self.streamImageView.contentMode = UIViewContentModeScaleAspectFit; + self.streamImageView.backgroundColor = UIColor.blackColor; + self.streamImageView.userInteractionEnabled = NO; + [root addSubview:self.streamImageView]; + self.overlayLabel = [[UILabel alloc] initWithFrame:CGRectZero]; self.overlayLabel.translatesAutoresizingMaskIntoConstraints = NO; self.overlayLabel.numberOfLines = 0; @@ -61,12 +74,20 @@ [self.motionManager startGyroUpdates]; } + self.streamViewportSize = CGSizeMake(2160.0, 1620.0); + self.streamScale = 2.0; + self.streamStatus = @"stream idle"; + NSURL *streamURL = [NSURL URLWithString:@"ws://192.168.1.66:8791/stream"]; + self.streamClient = [[EVEFrameStreamClient alloc] initWithURL:streamURL delegate:self]; + [self.streamClient connect]; + self.displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(frameTick:)]; [self.displayLink addToRunLoop:NSRunLoop.mainRunLoop forMode:NSRunLoopCommonModes]; } - (void)dealloc { [self.displayLink invalidate]; + [self.streamClient disconnect]; [self.motionManager stopAccelerometerUpdates]; [self.motionManager stopGyroUpdates]; } @@ -113,36 +134,99 @@ self.overlayLabel.text = [NSString stringWithFormat: @"EVE Canvas\n" - "native shell + UIKit text\n" + "CEF stream + native touch\n" + "%@\n" "points %.0fx%.0f pixels %.0fx%.0f @ %.1fx\n" + "stream %.0fx%.0f @ %.1fx\n" "fps %.1f touches %lu\n" "accel %+0.2f %+0.2f %+0.2f\n" "gyro %+0.2f %+0.2f %+0.2f", + self.streamStatus ?: @"stream", points.width, points.height, pixels.width, pixels.height, scale, + self.streamViewportSize.width, self.streamViewportSize.height, self.streamScale, self.filteredFPS, (unsigned long)self.touchCount, acceleration.x, acceleration.y, acceleration.z, rotation.x, rotation.y, rotation.z]; [self.overlayLabel sizeToFit]; } +- (CGPoint)streamPointForTouch:(UITouch *)touch { + CGPoint viewPoint = [touch locationInView:self.streamImageView]; + CGSize bounds = self.streamImageView.bounds.size; + CGSize viewport = self.streamViewportSize; + if (viewport.width <= 0 || viewport.height <= 0 || bounds.width <= 0 || bounds.height <= 0) { + return CGPointZero; + } + + CGFloat imageAspect = viewport.width / viewport.height; + CGFloat boundsAspect = bounds.width / bounds.height; + CGFloat drawWidth = bounds.width; + CGFloat drawHeight = bounds.height; + CGFloat offsetX = 0.0; + CGFloat offsetY = 0.0; + if (boundsAspect > imageAspect) { + drawWidth = bounds.height * imageAspect; + offsetX = (bounds.width - drawWidth) * 0.5; + } else { + drawHeight = bounds.width / imageAspect; + offsetY = (bounds.height - drawHeight) * 0.5; + } + + CGFloat x = (viewPoint.x - offsetX) * viewport.width / drawWidth; + CGFloat y = (viewPoint.y - offsetY) * viewport.height / drawHeight; + x = MIN(MAX(x, 0.0), viewport.width - 1.0); + y = MIN(MAX(y, 0.0), viewport.height - 1.0); + return CGPointMake(x, y); +} + +- (void)sendTouches:(NSSet *)touches event:(UIEvent *)event phase:(NSString *)phase { + (void)event; + UITouch *touch = touches.anyObject; + if (!touch) { + return; + } + CGPoint point = [self streamPointForTouch:touch]; + [self.streamClient sendPointerPhase:phase x:point.x y:point.y]; +} + - (void)touchesBegan:(NSSet *)touches withEvent:(UIEvent *)event { - (void)touches; self.touchCount = event.allTouches.count; + [self sendTouches:touches event:event phase:@"down"]; } - (void)touchesMoved:(NSSet *)touches withEvent:(UIEvent *)event { - (void)touches; self.touchCount = event.allTouches.count; + [self sendTouches:touches event:event phase:@"move"]; } - (void)touchesEnded:(NSSet *)touches withEvent:(UIEvent *)event { - (void)touches; self.touchCount = event.allTouches.count; + [self sendTouches:touches event:event phase:@"up"]; } - (void)touchesCancelled:(NSSet *)touches withEvent:(UIEvent *)event { - (void)touches; self.touchCount = event.allTouches.count; + [self sendTouches:touches event:event phase:@"up"]; +} + +- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveImage:(UIImage *)image { + (void)client; + self.streamImageView.image = image; +} + +- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveViewportWidth:(CGFloat)width height:(CGFloat)height scale:(CGFloat)scale { + (void)client; + if (width > 0 && height > 0) { + self.streamViewportSize = CGSizeMake(width, height); + } + if (scale > 0) { + self.streamScale = scale; + } +} + +- (void)frameStreamClient:(EVEFrameStreamClient *)client didChangeStatus:(NSString *)status { + (void)client; + self.streamStatus = status; } @end