Receive CEF dashboard stream

This commit is contained in:
Meta Vens 2026-05-25 20:43:56 +01:00
commit 4d49c820a7
6 changed files with 286 additions and 15 deletions

View file

@ -9,7 +9,8 @@ EveCanvas_FILES = \
Sources/main.m \
Sources/EVEAppDelegate.m \
Sources/EVEViewController.mm \
Sources/EVEGLView.mm
Sources/EVEGLView.mm \
Sources/EVEFrameStreamClient.m
EveCanvas_FRAMEWORKS = UIKit QuartzCore OpenGLES CoreMotion
EveCanvas_CFLAGS = -fobjc-arc -Wall -Wextra -DGLES_SILENCE_DEPRECATION

View file

@ -3,8 +3,10 @@
Fullscreen native render shell for EVE, the jailbroken iPad 8th generation test
device.
This is deliberately not a web app. UIKit owns the app/window lifecycle and the
text overlay. OpenGL ES 3.0 owns the render surface.
This is deliberately not a web app. UIKit owns the app/window lifecycle, the
streamed frame surface, touch capture, and the text overlay. OpenGL ES remains a
local render fallback, but the VoidBot dashboard path is a native frame stream
from Starfire.
## Objective
@ -15,21 +17,35 @@ the development machine.
- `EVEAppDelegate` creates one fullscreen `UIWindow`.
- `EVEViewController` installs:
- `EVEGLView`, a `CAEAGLLayer` backed OpenGL ES 3.0 render target.
- a native UIKit `UILabel` overlay for crisp Retina text.
- `EVEGLView`, a `CAEAGLLayer` backed OpenGL ES render target.
- a full-screen `UIImageView` that displays CEF relay frames.
- `EVEFrameStreamClient`, a native WebSocket client for frame/input transport.
- a native UIKit `UILabel` overlay for crisp Retina status text.
- `CADisplayLink` for frame ticking.
- `CMMotionManager` for accelerometer and gyro telemetry.
- `EVEGLView` clears the full drawable with a slow color pulse and presents the
renderbuffer every frame.
- `EVEFrameStreamClient` receives binary JPEG frames from the Starfire CEF relay
and sends touch events back as JSON viewport coordinates.
## Invariants
- Browser/PWA layout is not part of this path.
- UIKit is the text owner; OpenGL ES is the pixel owner.
- Browser/PWA layout is not part of the iPad path.
- Starfire CEF owns browser pixels; EveCanvas owns display and touch capture.
- UIKit is the current streamed-frame owner; OpenGL ES is fallback/local render.
- The status bar stays hidden.
- Sensor reads are display-only telemetry until a later input model owns them.
- CultMesh networking is not smuggled into the render shell yet.
## VoidBot CEF Stream
Start the Starfire relay from `E:\Projects\VoidBot`:
```powershell
npm run swarm:eve-cef-relay -- --width 2160 --height 1620 --scale 2 --port 8791
```
EveCanvas connects to `ws://192.168.1.66:8791/stream`, displays binary JPEG
frames, and returns touch events to the relay.
## Build Shape
The project is a Theos-style iOS application:

View file

@ -23,6 +23,11 @@
<string>1</string>
<key>LSRequiresIPhoneOS</key>
<true/>
<key>NSAppTransportSecurity</key>
<dict>
<key>NSAllowsArbitraryLoads</key>
<true/>
</dict>
<key>UIApplicationSupportsIndirectInputEvents</key>
<true/>
<key>UIRequiresFullScreen</key>

View file

@ -0,0 +1,20 @@
#import <UIKit/UIKit.h>
@protocol EVEFrameStreamClientDelegate;
@interface EVEFrameStreamClient : NSObject
- (instancetype)initWithURL:(NSURL *)url delegate:(id<EVEFrameStreamClientDelegate>)delegate;
- (void)connect;
- (void)disconnect;
- (void)sendPointerPhase:(NSString *)phase x:(CGFloat)x y:(CGFloat)y;
@end
@protocol EVEFrameStreamClientDelegate <NSObject>
- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveImage:(UIImage *)image;
- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveViewportWidth:(CGFloat)width height:(CGFloat)height scale:(CGFloat)scale;
- (void)frameStreamClient:(EVEFrameStreamClient *)client didChangeStatus:(NSString *)status;
@end

View file

@ -0,0 +1,145 @@
#import "EVEFrameStreamClient.h"
@interface EVEFrameStreamClient ()
@property(nonatomic, strong) NSURL *url;
@property(nonatomic, weak) id<EVEFrameStreamClientDelegate> delegate;
@property(nonatomic, strong) NSURLSession *session;
@property(nonatomic, strong) NSURLSessionWebSocketTask *task;
@property(nonatomic, assign) BOOL shouldReconnect;
@end
@implementation EVEFrameStreamClient
- (instancetype)initWithURL:(NSURL *)url delegate:(id<EVEFrameStreamClientDelegate>)delegate {
self = [super init];
if (!self) {
return nil;
}
self.url = url;
self.delegate = delegate;
self.session = [NSURLSession sessionWithConfiguration:NSURLSessionConfiguration.defaultSessionConfiguration];
self.shouldReconnect = YES;
return self;
}
- (void)connect {
self.shouldReconnect = YES;
[self.task cancelWithCloseCode:NSURLSessionWebSocketCloseCodeNormalClosure reason:nil];
self.task = [self.session webSocketTaskWithURL:self.url];
[self.task resume];
[self publishStatus:@"connecting"];
[self receiveNextMessage];
}
- (void)disconnect {
self.shouldReconnect = NO;
[self.task cancelWithCloseCode:NSURLSessionWebSocketCloseCodeNormalClosure reason:nil];
self.task = nil;
}
- (void)sendPointerPhase:(NSString *)phase x:(CGFloat)x y:(CGFloat)y {
if (!self.task) {
return;
}
NSDictionary *payload = @{
@"type": @"pointer",
@"phase": phase,
@"x": @(x),
@"y": @(y),
};
NSData *json = [NSJSONSerialization dataWithJSONObject:payload options:0 error:nil];
if (!json) {
return;
}
NSString *text = [[NSString alloc] initWithData:json encoding:NSUTF8StringEncoding];
NSURLSessionWebSocketMessage *message = [[NSURLSessionWebSocketMessage alloc] initWithString:text];
[self.task sendMessage:message completionHandler:^(NSError *error) {
if (error) {
[self publishStatus:@"input send failed"];
}
}];
}
- (void)receiveNextMessage {
__weak typeof(self) weakSelf = self;
[self.task receiveMessageWithCompletionHandler:^(NSURLSessionWebSocketMessage *message, NSError *error) {
__strong typeof(weakSelf) self = weakSelf;
if (!self) {
return;
}
if (error) {
[self publishStatus:@"stream disconnected"];
[self reconnectSoon];
return;
}
if (message.type == NSURLSessionWebSocketMessageTypeData) {
UIImage *image = [UIImage imageWithData:message.data];
if (image) {
dispatch_async(dispatch_get_main_queue(), ^{
[self.delegate frameStreamClient:self didReceiveImage:image];
});
}
} else if (message.type == NSURLSessionWebSocketMessageTypeString) {
[self handleTextMessage:message.string];
}
[self receiveNextMessage];
}];
}
- (void)handleTextMessage:(NSString *)text {
NSData *data = [text dataUsingEncoding:NSUTF8StringEncoding];
if (!data) {
return;
}
NSDictionary *message = [NSJSONSerialization JSONObjectWithData:data options:0 error:nil];
if (![message isKindOfClass:NSDictionary.class]) {
return;
}
if ([message[@"type"] isEqualToString:@"config"]) {
CGFloat width = [message[@"width"] doubleValue];
CGFloat height = [message[@"height"] doubleValue];
CGFloat scale = [message[@"DeviceScaleFactor"] doubleValue];
if (scale <= 0) {
scale = [message[@"deviceScaleFactor"] doubleValue];
}
dispatch_async(dispatch_get_main_queue(), ^{
[self.delegate frameStreamClient:self didReceiveViewportWidth:width height:height scale:scale];
[self.delegate frameStreamClient:self didChangeStatus:@"stream live"];
});
}
}
- (void)reconnectSoon {
if (!self.shouldReconnect) {
return;
}
dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(2.0 * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{
if (self.shouldReconnect) {
[self connect];
}
});
}
- (void)publishStatus:(NSString *)status {
dispatch_async(dispatch_get_main_queue(), ^{
[self.delegate frameStreamClient:self didChangeStatus:status];
});
}
- (void)dealloc {
[self disconnect];
[self.session invalidateAndCancel];
}
@end

View file

@ -3,17 +3,23 @@
#import <CoreMotion/CoreMotion.h>
#import <QuartzCore/QuartzCore.h>
#import "EVEFrameStreamClient.h"
#import "EVEGLView.h"
@interface EVEViewController ()
@interface EVEViewController () <EVEFrameStreamClientDelegate>
@property(nonatomic, strong) EVEGLView *glView;
@property(nonatomic, strong) UIImageView *streamImageView;
@property(nonatomic, strong) UILabel *overlayLabel;
@property(nonatomic, strong) CADisplayLink *displayLink;
@property(nonatomic, strong) CMMotionManager *motionManager;
@property(nonatomic, strong) EVEFrameStreamClient *streamClient;
@property(nonatomic, assign) NSTimeInterval previousTimestamp;
@property(nonatomic, assign) double filteredFPS;
@property(nonatomic, assign) NSUInteger touchCount;
@property(nonatomic, assign) CGSize streamViewportSize;
@property(nonatomic, assign) CGFloat streamScale;
@property(nonatomic, copy) NSString *streamStatus;
@end
@ -28,6 +34,13 @@
self.glView.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight;
[root addSubview:self.glView];
self.streamImageView = [[UIImageView alloc] initWithFrame:root.bounds];
self.streamImageView.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight;
self.streamImageView.contentMode = UIViewContentModeScaleAspectFit;
self.streamImageView.backgroundColor = UIColor.blackColor;
self.streamImageView.userInteractionEnabled = NO;
[root addSubview:self.streamImageView];
self.overlayLabel = [[UILabel alloc] initWithFrame:CGRectZero];
self.overlayLabel.translatesAutoresizingMaskIntoConstraints = NO;
self.overlayLabel.numberOfLines = 0;
@ -61,12 +74,20 @@
[self.motionManager startGyroUpdates];
}
self.streamViewportSize = CGSizeMake(2160.0, 1620.0);
self.streamScale = 2.0;
self.streamStatus = @"stream idle";
NSURL *streamURL = [NSURL URLWithString:@"ws://192.168.1.66:8791/stream"];
self.streamClient = [[EVEFrameStreamClient alloc] initWithURL:streamURL delegate:self];
[self.streamClient connect];
self.displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(frameTick:)];
[self.displayLink addToRunLoop:NSRunLoop.mainRunLoop forMode:NSRunLoopCommonModes];
}
- (void)dealloc {
[self.displayLink invalidate];
[self.streamClient disconnect];
[self.motionManager stopAccelerometerUpdates];
[self.motionManager stopGyroUpdates];
}
@ -113,36 +134,99 @@
self.overlayLabel.text = [NSString stringWithFormat:
@"EVE Canvas\n"
"native shell + UIKit text\n"
"CEF stream + native touch\n"
"%@\n"
"points %.0fx%.0f pixels %.0fx%.0f @ %.1fx\n"
"stream %.0fx%.0f @ %.1fx\n"
"fps %.1f touches %lu\n"
"accel %+0.2f %+0.2f %+0.2f\n"
"gyro %+0.2f %+0.2f %+0.2f",
self.streamStatus ?: @"stream",
points.width, points.height, pixels.width, pixels.height, scale,
self.streamViewportSize.width, self.streamViewportSize.height, self.streamScale,
self.filteredFPS, (unsigned long)self.touchCount,
acceleration.x, acceleration.y, acceleration.z,
rotation.x, rotation.y, rotation.z];
[self.overlayLabel sizeToFit];
}
- (CGPoint)streamPointForTouch:(UITouch *)touch {
CGPoint viewPoint = [touch locationInView:self.streamImageView];
CGSize bounds = self.streamImageView.bounds.size;
CGSize viewport = self.streamViewportSize;
if (viewport.width <= 0 || viewport.height <= 0 || bounds.width <= 0 || bounds.height <= 0) {
return CGPointZero;
}
CGFloat imageAspect = viewport.width / viewport.height;
CGFloat boundsAspect = bounds.width / bounds.height;
CGFloat drawWidth = bounds.width;
CGFloat drawHeight = bounds.height;
CGFloat offsetX = 0.0;
CGFloat offsetY = 0.0;
if (boundsAspect > imageAspect) {
drawWidth = bounds.height * imageAspect;
offsetX = (bounds.width - drawWidth) * 0.5;
} else {
drawHeight = bounds.width / imageAspect;
offsetY = (bounds.height - drawHeight) * 0.5;
}
CGFloat x = (viewPoint.x - offsetX) * viewport.width / drawWidth;
CGFloat y = (viewPoint.y - offsetY) * viewport.height / drawHeight;
x = MIN(MAX(x, 0.0), viewport.width - 1.0);
y = MIN(MAX(y, 0.0), viewport.height - 1.0);
return CGPointMake(x, y);
}
- (void)sendTouches:(NSSet<UITouch *> *)touches event:(UIEvent *)event phase:(NSString *)phase {
(void)event;
UITouch *touch = touches.anyObject;
if (!touch) {
return;
}
CGPoint point = [self streamPointForTouch:touch];
[self.streamClient sendPointerPhase:phase x:point.x y:point.y];
}
- (void)touchesBegan:(NSSet<UITouch *> *)touches withEvent:(UIEvent *)event {
(void)touches;
self.touchCount = event.allTouches.count;
[self sendTouches:touches event:event phase:@"down"];
}
- (void)touchesMoved:(NSSet<UITouch *> *)touches withEvent:(UIEvent *)event {
(void)touches;
self.touchCount = event.allTouches.count;
[self sendTouches:touches event:event phase:@"move"];
}
- (void)touchesEnded:(NSSet<UITouch *> *)touches withEvent:(UIEvent *)event {
(void)touches;
self.touchCount = event.allTouches.count;
[self sendTouches:touches event:event phase:@"up"];
}
- (void)touchesCancelled:(NSSet<UITouch *> *)touches withEvent:(UIEvent *)event {
(void)touches;
self.touchCount = event.allTouches.count;
[self sendTouches:touches event:event phase:@"up"];
}
- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveImage:(UIImage *)image {
(void)client;
self.streamImageView.image = image;
}
- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveViewportWidth:(CGFloat)width height:(CGFloat)height scale:(CGFloat)scale {
(void)client;
if (width > 0 && height > 0) {
self.streamViewportSize = CGSizeMake(width, height);
}
if (scale > 0) {
self.streamScale = scale;
}
}
- (void)frameStreamClient:(EVEFrameStreamClient *)client didChangeStatus:(NSString *)status {
(void)client;
self.streamStatus = status;
}
@end