GameCult/Thing
Watch
9
0
Fork
You've already forked Thing
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add native H264 EveCanvas stream decoder

Browse source
This commit is contained in:
Meta Vens 2026-05-25 22:46:05 +01:00
commit 897d73a98c
6 changed files with 294 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

5
Makefile
View file

@ -10,9 +10,10 @@ EveCanvas_FILES = \
Sources/EVEAppDelegate.m \
Sources/EVEViewController.mm \
Sources/EVEGLView.mm \
Sources/EVEFrameStreamClient.m
Sources/EVEFrameStreamClient.m \
Sources/EVEH264StreamDecoder.m
EveCanvas_FRAMEWORKS = UIKit QuartzCore OpenGLES CoreMotion
EveCanvas_FRAMEWORKS = UIKit QuartzCore OpenGLES CoreMotion AVFoundation CoreMedia CoreVideo VideoToolbox
EveCanvas_CFLAGS = -fobjc-arc -Wall -Wextra -DGLES_SILENCE_DEPRECATION
EveCanvas_CODESIGN_FLAGS = -Sentitlements.plist
EveCanvas_INSTALL_PATH = /Applications

3
Sources/EVEFrameStreamClient.h
View file

@ -5,6 +5,7 @@
@interface EVEFrameStreamClient : NSObject
- (instancetype)initWithURL:(NSURL *)url delegate:(id<EVEFrameStreamClientDelegate>)delegate;
- (instancetype)initWithURLs:(NSArray<NSURL *> *)urls delegate:(id<EVEFrameStreamClientDelegate>)delegate;
- (void)connect;
- (void)disconnect;
- (void)sendPointerPhase:(NSString *)phase x:(CGFloat)x y:(CGFloat)y;
@ -14,7 +15,9 @@
@protocol EVEFrameStreamClientDelegate <NSObject>
- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveImage:(UIImage *)image;
- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveVideoAccessUnit:(NSData *)data;
- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveViewportWidth:(CGFloat)width height:(CGFloat)height scale:(CGFloat)scale;
- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveCodec:(NSString *)codec;
- (void)frameStreamClient:(EVEFrameStreamClient *)client didChangeStatus:(NSString *)status;
@end

41
Sources/EVEFrameStreamClient.m
View file

@ -2,7 +2,8 @@
@interface EVEFrameStreamClient ()
@property(nonatomic, strong) NSURL *url;
@property(nonatomic, copy) NSArray<NSURL *> *urls;
@property(nonatomic, assign) NSUInteger urlIndex;
@property(nonatomic, weak) id<EVEFrameStreamClientDelegate> delegate;
@property(nonatomic, strong) NSURLSession *session;
@property(nonatomic, strong) NSURLSessionWebSocketTask *task;
@ -13,12 +14,16 @@
@implementation EVEFrameStreamClient
- (instancetype)initWithURL:(NSURL *)url delegate:(id<EVEFrameStreamClientDelegate>)delegate {
return [self initWithURLs:url ? @[url] : @[] delegate:delegate];
}
- (instancetype)initWithURLs:(NSArray<NSURL *> *)urls delegate:(id<EVEFrameStreamClientDelegate>)delegate {
self = [super init];
if (!self) {
return nil;
}
self.url = url;
self.urls = urls.count > 0 ? urls : @[];
self.delegate = delegate;
self.session = [NSURLSession sessionWithConfiguration:NSURLSessionConfiguration.defaultSessionConfiguration];
self.shouldReconnect = YES;
@ -28,9 +33,15 @@
- (void)connect {
self.shouldReconnect = YES;
[self.task cancelWithCloseCode:NSURLSessionWebSocketCloseCodeNormalClosure reason:nil];
self.task = [self.session webSocketTaskWithURL:self.url];
NSURL *url = self.urls.count > 0 ? self.urls[self.urlIndex % self.urls.count] : nil;
if (!url) {
[self publishStatus:@"stream URL missing"];
return;
}
self.task = [self.session webSocketTaskWithURL:url];
[self.task resume];
[self publishStatus:@"connecting"];
[self publishStatus:[NSString stringWithFormat:@"connecting %@", url.host ?: @""]];
[self receiveNextMessage];
}
@ -80,10 +91,18 @@
}
if (message.type == NSURLSessionWebSocketMessageTypeData) {
UIImage *image = [UIImage imageWithData:message.data];
if (image) {
const uint8_t *bytes = message.data.bytes;
BOOL isJpeg = message.data.length > 2 && bytes[0] == 0xff && bytes[1] == 0xd8;
if (isJpeg) {
UIImage *image = [UIImage imageWithData:message.data];
if (image) {
dispatch_async(dispatch_get_main_queue(), ^{
[self.delegate frameStreamClient:self didReceiveImage:image];
});
}
} else {
dispatch_async(dispatch_get_main_queue(), ^{
[self.delegate frameStreamClient:self didReceiveImage:image];
[self.delegate frameStreamClient:self didReceiveVideoAccessUnit:message.data];
});
}
} else if (message.type == NSURLSessionWebSocketMessageTypeString) {
@ -114,6 +133,10 @@
}
dispatch_async(dispatch_get_main_queue(), ^{
[self.delegate frameStreamClient:self didReceiveViewportWidth:width height:height scale:scale];
NSString *codec = message[@"codec"];
if ([codec isKindOfClass:NSString.class]) {
[self.delegate frameStreamClient:self didReceiveCodec:codec];
}
[self.delegate frameStreamClient:self didChangeStatus:@"stream live"];
});
}
@ -124,6 +147,10 @@
return;
}
if (self.urls.count > 1) {
self.urlIndex = (self.urlIndex + 1) % self.urls.count;
}
dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(2.0 * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{
if (self.shouldReconnect) {
[self connect];

15
Sources/EVEH264StreamDecoder.h Normal file
View file

@ -0,0 +1,15 @@
#import <AVFoundation/AVFoundation.h>
#import <Foundation/Foundation.h>
NS_ASSUME_NONNULL_BEGIN
@interface EVEH264StreamDecoder : NSObject
@property(nonatomic, readonly) AVSampleBufferDisplayLayer *displayLayer;
- (void)reset;
- (void)consumeAnnexBAccessUnit:(NSData *)data;
@end
NS_ASSUME_NONNULL_END

205
Sources/EVEH264StreamDecoder.m Normal file
View file

@ -0,0 +1,205 @@
#import "EVEH264StreamDecoder.h"
#import <CoreMedia/CoreMedia.h>
#import <UIKit/UIKit.h>
#import <VideoToolbox/VideoToolbox.h>
@interface EVEH264StreamDecoder ()
@property(nonatomic, strong) AVSampleBufferDisplayLayer *displayLayer;
@property(nonatomic) CMVideoFormatDescriptionRef formatDescription;
@property(nonatomic, strong) NSData *sps;
@property(nonatomic, strong) NSData *pps;
@property(nonatomic, assign) int64_t frameIndex;
@end
@implementation EVEH264StreamDecoder
- (instancetype)init {
self = [super init];
if (!self) {
return nil;
}
_displayLayer = [[AVSampleBufferDisplayLayer alloc] init];
_displayLayer.videoGravity = AVLayerVideoGravityResizeAspect;
_displayLayer.backgroundColor = UIColor.blackColor.CGColor;
return self;
}
- (void)dealloc {
[self reset];
}
- (void)reset {
if (_formatDescription) {
CFRelease(_formatDescription);
_formatDescription = NULL;
}
self.sps = nil;
self.pps = nil;
self.frameIndex = 0;
[self.displayLayer flushAndRemoveImage];
}
- (void)consumeAnnexBAccessUnit:(NSData *)data {
NSArray<NSData *> *nals = [self parseAnnexB:data];
if (nals.count == 0) {
return;
}
NSMutableData *avcc = [NSMutableData data];
BOOL hasSlice = NO;
for (NSData *nal in nals) {
if (nal.length == 0) {
continue;
}
const uint8_t *bytes = nal.bytes;
uint8_t nalType = bytes[0] & 0x1f;
if (nalType == 7) {
self.sps = nal;
[self rebuildFormatDescriptionIfReady];
continue;
}
if (nalType == 8) {
self.pps = nal;
[self rebuildFormatDescriptionIfReady];
continue;
}
if (nalType == 1 || nalType == 5) {
hasSlice = YES;
}
uint32_t length = CFSwapInt32HostToBig((uint32_t)nal.length);
[avcc appendBytes:&length length:sizeof(length)];
[avcc appendData:nal];
}
if (!hasSlice || !_formatDescription || avcc.length == 0) {
return;
}
CMBlockBufferRef blockBuffer = NULL;
OSStatus status = CMBlockBufferCreateWithMemoryBlock(
kCFAllocatorDefault,
NULL,
avcc.length,
kCFAllocatorDefault,
NULL,
0,
avcc.length,
0,
&blockBuffer);
if (status != noErr || !blockBuffer) {
return;
}
status = CMBlockBufferReplaceDataBytes(avcc.bytes, blockBuffer, 0, avcc.length);
if (status != noErr) {
CFRelease(blockBuffer);
return;
}
CMSampleTimingInfo timing = {
.duration = CMTimeMake(1, 60),
.presentationTimeStamp = CMTimeMake(self.frameIndex++, 60),
.decodeTimeStamp = kCMTimeInvalid,
};
size_t sampleSize = avcc.length;
CMSampleBufferRef sampleBuffer = NULL;
status = CMSampleBufferCreateReady(
kCFAllocatorDefault,
blockBuffer,
_formatDescription,
1,
1,
&timing,
1,
&sampleSize,
&sampleBuffer);
CFRelease(blockBuffer);
if (status != noErr || !sampleBuffer) {
return;
}
CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, YES);
if (attachments && CFArrayGetCount(attachments) > 0) {
CFMutableDictionaryRef attachment = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachments, 0);
CFDictionarySetValue(attachment, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue);
}
if (self.displayLayer.status == AVQueuedSampleBufferRenderingStatusFailed) {
[self.displayLayer flush];
}
[self.displayLayer enqueueSampleBuffer:sampleBuffer];
CFRelease(sampleBuffer);
}
- (void)rebuildFormatDescriptionIfReady {
if (!self.sps || !self.pps) {
return;
}
if (_formatDescription) {
CFRelease(_formatDescription);
_formatDescription = NULL;
}
const uint8_t *parameterSets[] = { self.sps.bytes, self.pps.bytes };
size_t parameterSetSizes[] = { self.sps.length, self.pps.length };
CMVideoFormatDescriptionCreateFromH264ParameterSets(
kCFAllocatorDefault,
2,
parameterSets,
parameterSetSizes,
4,
&_formatDescription);
}
- (NSArray<NSData *> *)parseAnnexB:(NSData *)data {
NSMutableArray<NSData *> *nals = [NSMutableArray array];
const uint8_t *bytes = data.bytes;
NSUInteger length = data.length;
NSUInteger cursor = 0;
while (cursor + 3 < length) {
NSUInteger start = [self findStartCode:bytes length:length offset:cursor startCodeLength:NULL];
if (start == NSNotFound) {
break;
}
NSUInteger startCodeLength = 0;
[self findStartCode:bytes length:length offset:start startCodeLength:&startCodeLength];
NSUInteger payloadStart = start + startCodeLength;
NSUInteger next = [self findStartCode:bytes length:length offset:payloadStart startCodeLength:NULL];
NSUInteger payloadEnd = next == NSNotFound ? length : next;
if (payloadEnd > payloadStart) {
[nals addObject:[data subdataWithRange:NSMakeRange(payloadStart, payloadEnd - payloadStart)]];
}
cursor = payloadEnd;
}
return nals;
}
- (NSUInteger)findStartCode:(const uint8_t *)bytes length:(NSUInteger)length offset:(NSUInteger)offset startCodeLength:(NSUInteger *)startCodeLength {
for (NSUInteger index = offset; index + 3 < length; index++) {
if (bytes[index] == 0 && bytes[index + 1] == 0 && bytes[index + 2] == 1) {
if (startCodeLength) {
*startCodeLength = 3;
}
return index;
}
if (index + 4 < length && bytes[index] == 0 && bytes[index + 1] == 0 && bytes[index + 2] == 0 && bytes[index + 3] == 1) {
if (startCodeLength) {
*startCodeLength = 4;
}
return index;
}
}
return NSNotFound;
}
@end

36
Sources/EVEViewController.mm
View file

@ -5,11 +5,13 @@
#import "EVEFrameStreamClient.h"
#import "EVEGLView.h"
#import "EVEH264StreamDecoder.h"
@interface EVEViewController () <EVEFrameStreamClientDelegate>
@property(nonatomic, strong) EVEGLView *glView;
@property(nonatomic, strong) UIImageView *streamImageView;
@property(nonatomic, strong) EVEH264StreamDecoder *videoDecoder;
@property(nonatomic, strong) UILabel *overlayLabel;
@property(nonatomic, strong) CADisplayLink *displayLink;
@property(nonatomic, strong) CMMotionManager *motionManager;
@ -20,6 +22,7 @@
@property(nonatomic, assign) CGSize streamViewportSize;
@property(nonatomic, assign) CGFloat streamScale;
@property(nonatomic, copy) NSString *streamStatus;
@property(nonatomic, copy) NSString *streamCodec;
@end
@ -41,6 +44,11 @@
self.streamImageView.userInteractionEnabled = NO;
[root addSubview:self.streamImageView];
self.videoDecoder = [[EVEH264StreamDecoder alloc] init];
self.videoDecoder.displayLayer.frame = root.bounds;
self.videoDecoder.displayLayer.hidden = YES;
[root.layer addSublayer:self.videoDecoder.displayLayer];
self.overlayLabel = [[UILabel alloc] initWithFrame:CGRectZero];
self.overlayLabel.translatesAutoresizingMaskIntoConstraints = NO;
self.overlayLabel.numberOfLines = 0;
@ -77,8 +85,12 @@
self.streamViewportSize = CGSizeMake(1620.0, 2160.0);
self.streamScale = 2.0;
self.streamStatus = @"stream idle";
NSURL *streamURL = [NSURL URLWithString:@"ws://192.168.1.66:8792/stream"];
self.streamClient = [[EVEFrameStreamClient alloc] initWithURL:streamURL delegate:self];
self.streamCodec = @"jpeg";
NSArray<NSURL *> *streamURLs = @[
[NSURL URLWithString:@"ws://127.0.0.1:8792/stream"],
[NSURL URLWithString:@"ws://192.168.1.66:8792/stream"],
];
self.streamClient = [[EVEFrameStreamClient alloc] initWithURLs:streamURLs delegate:self];
[self.streamClient connect];
self.displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(frameTick:)];
@ -107,6 +119,7 @@
- (void)viewDidLayoutSubviews {
[super viewDidLayoutSubviews];
[self.glView resizeDrawableIfNeeded];
self.videoDecoder.displayLayer.frame = self.view.bounds;
}
- (void)frameTick:(CADisplayLink *)link {
@ -136,12 +149,14 @@
@"EVE Canvas\n"
"CEF stream + native touch\n"
"%@\n"
"codec %@\n"
"points %.0fx%.0f pixels %.0fx%.0f @ %.1fx\n"
"stream %.0fx%.0f @ %.1fx\n"
"fps %.1f touches %lu\n"
"accel %+0.2f %+0.2f %+0.2f\n"
"gyro %+0.2f %+0.2f %+0.2f",
self.streamStatus ?: @"stream",
self.streamCodec ?: @"unknown",
points.width, points.height, pixels.width, pixels.height, scale,
self.streamViewportSize.width, self.streamViewportSize.height, self.streamScale,
self.filteredFPS, (unsigned long)self.touchCount,
@ -211,9 +226,18 @@
- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveImage:(UIImage *)image {
(void)client;
self.videoDecoder.displayLayer.hidden = YES;
self.streamImageView.hidden = NO;
self.streamImageView.image = image;
}
- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveVideoAccessUnit:(NSData *)data {
(void)client;
self.streamImageView.hidden = YES;
self.videoDecoder.displayLayer.hidden = NO;
[self.videoDecoder consumeAnnexBAccessUnit:data];
}
- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveViewportWidth:(CGFloat)width height:(CGFloat)height scale:(CGFloat)scale {
(void)client;
if (width > 0 && height > 0) {
@ -224,6 +248,14 @@
}
}
- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveCodec:(NSString *)codec {
(void)client;
if (![codec isEqualToString:self.streamCodec]) {
[self.videoDecoder reset];
}
self.streamCodec = codec;
}
- (void)frameStreamClient:(EVEFrameStreamClient *)client didChangeStatus:(NSString *)status {
(void)client;
self.streamStatus = status;