Receive CEF dashboard stream
This commit is contained in:
parent
beaed6bc7e
commit
4d49c820a7
6 changed files with 286 additions and 15 deletions
3
Makefile
3
Makefile
|
|
@ -9,7 +9,8 @@ EveCanvas_FILES = \
|
|||
Sources/main.m \
|
||||
Sources/EVEAppDelegate.m \
|
||||
Sources/EVEViewController.mm \
|
||||
Sources/EVEGLView.mm
|
||||
Sources/EVEGLView.mm \
|
||||
Sources/EVEFrameStreamClient.m
|
||||
|
||||
EveCanvas_FRAMEWORKS = UIKit QuartzCore OpenGLES CoreMotion
|
||||
EveCanvas_CFLAGS = -fobjc-arc -Wall -Wextra -DGLES_SILENCE_DEPRECATION
|
||||
|
|
|
|||
32
README.md
32
README.md
|
|
@ -3,8 +3,10 @@
|
|||
Fullscreen native render shell for EVE, the jailbroken iPad 8th generation test
|
||||
device.
|
||||
|
||||
This is deliberately not a web app. UIKit owns the app/window lifecycle and the
|
||||
text overlay. OpenGL ES 3.0 owns the render surface.
|
||||
This is deliberately not a web app. UIKit owns the app/window lifecycle, the
|
||||
streamed frame surface, touch capture, and the text overlay. OpenGL ES remains a
|
||||
local render fallback, but the VoidBot dashboard path is a native frame stream
|
||||
from Starfire.
|
||||
|
||||
## Objective
|
||||
|
||||
|
|
@ -15,21 +17,35 @@ the development machine.
|
|||
|
||||
- `EVEAppDelegate` creates one fullscreen `UIWindow`.
|
||||
- `EVEViewController` installs:
|
||||
- `EVEGLView`, a `CAEAGLLayer` backed OpenGL ES 3.0 render target.
|
||||
- a native UIKit `UILabel` overlay for crisp Retina text.
|
||||
- `EVEGLView`, a `CAEAGLLayer` backed OpenGL ES render target.
|
||||
- a full-screen `UIImageView` that displays CEF relay frames.
|
||||
- `EVEFrameStreamClient`, a native WebSocket client for frame/input transport.
|
||||
- a native UIKit `UILabel` overlay for crisp Retina status text.
|
||||
- `CADisplayLink` for frame ticking.
|
||||
- `CMMotionManager` for accelerometer and gyro telemetry.
|
||||
- `EVEGLView` clears the full drawable with a slow color pulse and presents the
|
||||
renderbuffer every frame.
|
||||
- `EVEFrameStreamClient` receives binary JPEG frames from the Starfire CEF relay
|
||||
and sends touch events back as JSON viewport coordinates.
|
||||
|
||||
## Invariants
|
||||
|
||||
- Browser/PWA layout is not part of this path.
|
||||
- UIKit is the text owner; OpenGL ES is the pixel owner.
|
||||
- Browser/PWA layout is not part of the iPad path.
|
||||
- Starfire CEF owns browser pixels; EveCanvas owns display and touch capture.
|
||||
- UIKit is the current streamed-frame owner; OpenGL ES is fallback/local render.
|
||||
- The status bar stays hidden.
|
||||
- Sensor reads are display-only telemetry until a later input model owns them.
|
||||
- CultMesh networking is not smuggled into the render shell yet.
|
||||
|
||||
## VoidBot CEF Stream
|
||||
|
||||
Start the Starfire relay from `E:\Projects\VoidBot`:
|
||||
|
||||
```powershell
|
||||
npm run swarm:eve-cef-relay -- --width 2160 --height 1620 --scale 2 --port 8791
|
||||
```
|
||||
|
||||
EveCanvas connects to `ws://192.168.1.66:8791/stream`, displays binary JPEG
|
||||
frames, and returns touch events to the relay.
|
||||
|
||||
## Build Shape
|
||||
|
||||
The project is a Theos-style iOS application:
|
||||
|
|
|
|||
|
|
@ -23,6 +23,11 @@
|
|||
<string>1</string>
|
||||
<key>LSRequiresIPhoneOS</key>
|
||||
<true/>
|
||||
<key>NSAppTransportSecurity</key>
|
||||
<dict>
|
||||
<key>NSAllowsArbitraryLoads</key>
|
||||
<true/>
|
||||
</dict>
|
||||
<key>UIApplicationSupportsIndirectInputEvents</key>
|
||||
<true/>
|
||||
<key>UIRequiresFullScreen</key>
|
||||
|
|
|
|||
20
Sources/EVEFrameStreamClient.h
Normal file
20
Sources/EVEFrameStreamClient.h
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
#import <UIKit/UIKit.h>
|
||||
|
||||
@protocol EVEFrameStreamClientDelegate;
|
||||
|
||||
@interface EVEFrameStreamClient : NSObject
|
||||
|
||||
- (instancetype)initWithURL:(NSURL *)url delegate:(id<EVEFrameStreamClientDelegate>)delegate;
|
||||
- (void)connect;
|
||||
- (void)disconnect;
|
||||
- (void)sendPointerPhase:(NSString *)phase x:(CGFloat)x y:(CGFloat)y;
|
||||
|
||||
@end
|
||||
|
||||
@protocol EVEFrameStreamClientDelegate <NSObject>
|
||||
|
||||
- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveImage:(UIImage *)image;
|
||||
- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveViewportWidth:(CGFloat)width height:(CGFloat)height scale:(CGFloat)scale;
|
||||
- (void)frameStreamClient:(EVEFrameStreamClient *)client didChangeStatus:(NSString *)status;
|
||||
|
||||
@end
|
||||
145
Sources/EVEFrameStreamClient.m
Normal file
145
Sources/EVEFrameStreamClient.m
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
#import "EVEFrameStreamClient.h"
|
||||
|
||||
@interface EVEFrameStreamClient ()
|
||||
|
||||
@property(nonatomic, strong) NSURL *url;
|
||||
@property(nonatomic, weak) id<EVEFrameStreamClientDelegate> delegate;
|
||||
@property(nonatomic, strong) NSURLSession *session;
|
||||
@property(nonatomic, strong) NSURLSessionWebSocketTask *task;
|
||||
@property(nonatomic, assign) BOOL shouldReconnect;
|
||||
|
||||
@end
|
||||
|
||||
@implementation EVEFrameStreamClient
|
||||
|
||||
- (instancetype)initWithURL:(NSURL *)url delegate:(id<EVEFrameStreamClientDelegate>)delegate {
|
||||
self = [super init];
|
||||
if (!self) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
self.url = url;
|
||||
self.delegate = delegate;
|
||||
self.session = [NSURLSession sessionWithConfiguration:NSURLSessionConfiguration.defaultSessionConfiguration];
|
||||
self.shouldReconnect = YES;
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)connect {
|
||||
self.shouldReconnect = YES;
|
||||
[self.task cancelWithCloseCode:NSURLSessionWebSocketCloseCodeNormalClosure reason:nil];
|
||||
self.task = [self.session webSocketTaskWithURL:self.url];
|
||||
[self.task resume];
|
||||
[self publishStatus:@"connecting"];
|
||||
[self receiveNextMessage];
|
||||
}
|
||||
|
||||
- (void)disconnect {
|
||||
self.shouldReconnect = NO;
|
||||
[self.task cancelWithCloseCode:NSURLSessionWebSocketCloseCodeNormalClosure reason:nil];
|
||||
self.task = nil;
|
||||
}
|
||||
|
||||
- (void)sendPointerPhase:(NSString *)phase x:(CGFloat)x y:(CGFloat)y {
|
||||
if (!self.task) {
|
||||
return;
|
||||
}
|
||||
|
||||
NSDictionary *payload = @{
|
||||
@"type": @"pointer",
|
||||
@"phase": phase,
|
||||
@"x": @(x),
|
||||
@"y": @(y),
|
||||
};
|
||||
NSData *json = [NSJSONSerialization dataWithJSONObject:payload options:0 error:nil];
|
||||
if (!json) {
|
||||
return;
|
||||
}
|
||||
|
||||
NSString *text = [[NSString alloc] initWithData:json encoding:NSUTF8StringEncoding];
|
||||
NSURLSessionWebSocketMessage *message = [[NSURLSessionWebSocketMessage alloc] initWithString:text];
|
||||
[self.task sendMessage:message completionHandler:^(NSError *error) {
|
||||
if (error) {
|
||||
[self publishStatus:@"input send failed"];
|
||||
}
|
||||
}];
|
||||
}
|
||||
|
||||
- (void)receiveNextMessage {
|
||||
__weak typeof(self) weakSelf = self;
|
||||
[self.task receiveMessageWithCompletionHandler:^(NSURLSessionWebSocketMessage *message, NSError *error) {
|
||||
__strong typeof(weakSelf) self = weakSelf;
|
||||
if (!self) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (error) {
|
||||
[self publishStatus:@"stream disconnected"];
|
||||
[self reconnectSoon];
|
||||
return;
|
||||
}
|
||||
|
||||
if (message.type == NSURLSessionWebSocketMessageTypeData) {
|
||||
UIImage *image = [UIImage imageWithData:message.data];
|
||||
if (image) {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
[self.delegate frameStreamClient:self didReceiveImage:image];
|
||||
});
|
||||
}
|
||||
} else if (message.type == NSURLSessionWebSocketMessageTypeString) {
|
||||
[self handleTextMessage:message.string];
|
||||
}
|
||||
|
||||
[self receiveNextMessage];
|
||||
}];
|
||||
}
|
||||
|
||||
- (void)handleTextMessage:(NSString *)text {
|
||||
NSData *data = [text dataUsingEncoding:NSUTF8StringEncoding];
|
||||
if (!data) {
|
||||
return;
|
||||
}
|
||||
|
||||
NSDictionary *message = [NSJSONSerialization JSONObjectWithData:data options:0 error:nil];
|
||||
if (![message isKindOfClass:NSDictionary.class]) {
|
||||
return;
|
||||
}
|
||||
|
||||
if ([message[@"type"] isEqualToString:@"config"]) {
|
||||
CGFloat width = [message[@"width"] doubleValue];
|
||||
CGFloat height = [message[@"height"] doubleValue];
|
||||
CGFloat scale = [message[@"DeviceScaleFactor"] doubleValue];
|
||||
if (scale <= 0) {
|
||||
scale = [message[@"deviceScaleFactor"] doubleValue];
|
||||
}
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
[self.delegate frameStreamClient:self didReceiveViewportWidth:width height:height scale:scale];
|
||||
[self.delegate frameStreamClient:self didChangeStatus:@"stream live"];
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
- (void)reconnectSoon {
|
||||
if (!self.shouldReconnect) {
|
||||
return;
|
||||
}
|
||||
|
||||
dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(2.0 * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{
|
||||
if (self.shouldReconnect) {
|
||||
[self connect];
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
- (void)publishStatus:(NSString *)status {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
[self.delegate frameStreamClient:self didChangeStatus:status];
|
||||
});
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
[self disconnect];
|
||||
[self.session invalidateAndCancel];
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -3,17 +3,23 @@
|
|||
#import <CoreMotion/CoreMotion.h>
|
||||
#import <QuartzCore/QuartzCore.h>
|
||||
|
||||
#import "EVEFrameStreamClient.h"
|
||||
#import "EVEGLView.h"
|
||||
|
||||
@interface EVEViewController ()
|
||||
@interface EVEViewController () <EVEFrameStreamClientDelegate>
|
||||
|
||||
@property(nonatomic, strong) EVEGLView *glView;
|
||||
@property(nonatomic, strong) UIImageView *streamImageView;
|
||||
@property(nonatomic, strong) UILabel *overlayLabel;
|
||||
@property(nonatomic, strong) CADisplayLink *displayLink;
|
||||
@property(nonatomic, strong) CMMotionManager *motionManager;
|
||||
@property(nonatomic, strong) EVEFrameStreamClient *streamClient;
|
||||
@property(nonatomic, assign) NSTimeInterval previousTimestamp;
|
||||
@property(nonatomic, assign) double filteredFPS;
|
||||
@property(nonatomic, assign) NSUInteger touchCount;
|
||||
@property(nonatomic, assign) CGSize streamViewportSize;
|
||||
@property(nonatomic, assign) CGFloat streamScale;
|
||||
@property(nonatomic, copy) NSString *streamStatus;
|
||||
|
||||
@end
|
||||
|
||||
|
|
@ -28,6 +34,13 @@
|
|||
self.glView.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight;
|
||||
[root addSubview:self.glView];
|
||||
|
||||
self.streamImageView = [[UIImageView alloc] initWithFrame:root.bounds];
|
||||
self.streamImageView.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight;
|
||||
self.streamImageView.contentMode = UIViewContentModeScaleAspectFit;
|
||||
self.streamImageView.backgroundColor = UIColor.blackColor;
|
||||
self.streamImageView.userInteractionEnabled = NO;
|
||||
[root addSubview:self.streamImageView];
|
||||
|
||||
self.overlayLabel = [[UILabel alloc] initWithFrame:CGRectZero];
|
||||
self.overlayLabel.translatesAutoresizingMaskIntoConstraints = NO;
|
||||
self.overlayLabel.numberOfLines = 0;
|
||||
|
|
@ -61,12 +74,20 @@
|
|||
[self.motionManager startGyroUpdates];
|
||||
}
|
||||
|
||||
self.streamViewportSize = CGSizeMake(2160.0, 1620.0);
|
||||
self.streamScale = 2.0;
|
||||
self.streamStatus = @"stream idle";
|
||||
NSURL *streamURL = [NSURL URLWithString:@"ws://192.168.1.66:8791/stream"];
|
||||
self.streamClient = [[EVEFrameStreamClient alloc] initWithURL:streamURL delegate:self];
|
||||
[self.streamClient connect];
|
||||
|
||||
self.displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(frameTick:)];
|
||||
[self.displayLink addToRunLoop:NSRunLoop.mainRunLoop forMode:NSRunLoopCommonModes];
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
[self.displayLink invalidate];
|
||||
[self.streamClient disconnect];
|
||||
[self.motionManager stopAccelerometerUpdates];
|
||||
[self.motionManager stopGyroUpdates];
|
||||
}
|
||||
|
|
@ -113,36 +134,99 @@
|
|||
|
||||
self.overlayLabel.text = [NSString stringWithFormat:
|
||||
@"EVE Canvas\n"
|
||||
"native shell + UIKit text\n"
|
||||
"CEF stream + native touch\n"
|
||||
"%@\n"
|
||||
"points %.0fx%.0f pixels %.0fx%.0f @ %.1fx\n"
|
||||
"stream %.0fx%.0f @ %.1fx\n"
|
||||
"fps %.1f touches %lu\n"
|
||||
"accel %+0.2f %+0.2f %+0.2f\n"
|
||||
"gyro %+0.2f %+0.2f %+0.2f",
|
||||
self.streamStatus ?: @"stream",
|
||||
points.width, points.height, pixels.width, pixels.height, scale,
|
||||
self.streamViewportSize.width, self.streamViewportSize.height, self.streamScale,
|
||||
self.filteredFPS, (unsigned long)self.touchCount,
|
||||
acceleration.x, acceleration.y, acceleration.z,
|
||||
rotation.x, rotation.y, rotation.z];
|
||||
[self.overlayLabel sizeToFit];
|
||||
}
|
||||
|
||||
- (CGPoint)streamPointForTouch:(UITouch *)touch {
|
||||
CGPoint viewPoint = [touch locationInView:self.streamImageView];
|
||||
CGSize bounds = self.streamImageView.bounds.size;
|
||||
CGSize viewport = self.streamViewportSize;
|
||||
if (viewport.width <= 0 || viewport.height <= 0 || bounds.width <= 0 || bounds.height <= 0) {
|
||||
return CGPointZero;
|
||||
}
|
||||
|
||||
CGFloat imageAspect = viewport.width / viewport.height;
|
||||
CGFloat boundsAspect = bounds.width / bounds.height;
|
||||
CGFloat drawWidth = bounds.width;
|
||||
CGFloat drawHeight = bounds.height;
|
||||
CGFloat offsetX = 0.0;
|
||||
CGFloat offsetY = 0.0;
|
||||
if (boundsAspect > imageAspect) {
|
||||
drawWidth = bounds.height * imageAspect;
|
||||
offsetX = (bounds.width - drawWidth) * 0.5;
|
||||
} else {
|
||||
drawHeight = bounds.width / imageAspect;
|
||||
offsetY = (bounds.height - drawHeight) * 0.5;
|
||||
}
|
||||
|
||||
CGFloat x = (viewPoint.x - offsetX) * viewport.width / drawWidth;
|
||||
CGFloat y = (viewPoint.y - offsetY) * viewport.height / drawHeight;
|
||||
x = MIN(MAX(x, 0.0), viewport.width - 1.0);
|
||||
y = MIN(MAX(y, 0.0), viewport.height - 1.0);
|
||||
return CGPointMake(x, y);
|
||||
}
|
||||
|
||||
- (void)sendTouches:(NSSet<UITouch *> *)touches event:(UIEvent *)event phase:(NSString *)phase {
|
||||
(void)event;
|
||||
UITouch *touch = touches.anyObject;
|
||||
if (!touch) {
|
||||
return;
|
||||
}
|
||||
CGPoint point = [self streamPointForTouch:touch];
|
||||
[self.streamClient sendPointerPhase:phase x:point.x y:point.y];
|
||||
}
|
||||
|
||||
- (void)touchesBegan:(NSSet<UITouch *> *)touches withEvent:(UIEvent *)event {
|
||||
(void)touches;
|
||||
self.touchCount = event.allTouches.count;
|
||||
[self sendTouches:touches event:event phase:@"down"];
|
||||
}
|
||||
|
||||
- (void)touchesMoved:(NSSet<UITouch *> *)touches withEvent:(UIEvent *)event {
|
||||
(void)touches;
|
||||
self.touchCount = event.allTouches.count;
|
||||
[self sendTouches:touches event:event phase:@"move"];
|
||||
}
|
||||
|
||||
- (void)touchesEnded:(NSSet<UITouch *> *)touches withEvent:(UIEvent *)event {
|
||||
(void)touches;
|
||||
self.touchCount = event.allTouches.count;
|
||||
[self sendTouches:touches event:event phase:@"up"];
|
||||
}
|
||||
|
||||
- (void)touchesCancelled:(NSSet<UITouch *> *)touches withEvent:(UIEvent *)event {
|
||||
(void)touches;
|
||||
self.touchCount = event.allTouches.count;
|
||||
[self sendTouches:touches event:event phase:@"up"];
|
||||
}
|
||||
|
||||
- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveImage:(UIImage *)image {
|
||||
(void)client;
|
||||
self.streamImageView.image = image;
|
||||
}
|
||||
|
||||
- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveViewportWidth:(CGFloat)width height:(CGFloat)height scale:(CGFloat)scale {
|
||||
(void)client;
|
||||
if (width > 0 && height > 0) {
|
||||
self.streamViewportSize = CGSizeMake(width, height);
|
||||
}
|
||||
if (scale > 0) {
|
||||
self.streamScale = scale;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)frameStreamClient:(EVEFrameStreamClient *)client didChangeStatus:(NSString *)status {
|
||||
(void)client;
|
||||
self.streamStatus = status;
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
|
|||
Loading…
Reference in a new issue