diff --git a/Makefile b/Makefile index 38612ac..b98e69e 100644 --- a/Makefile +++ b/Makefile @@ -10,9 +10,10 @@ EveCanvas_FILES = \ Sources/EVEAppDelegate.m \ Sources/EVEViewController.mm \ Sources/EVEGLView.mm \ - Sources/EVEFrameStreamClient.m + Sources/EVEFrameStreamClient.m \ + Sources/EVEH264StreamDecoder.m -EveCanvas_FRAMEWORKS = UIKit QuartzCore OpenGLES CoreMotion +EveCanvas_FRAMEWORKS = UIKit QuartzCore OpenGLES CoreMotion AVFoundation CoreMedia CoreVideo VideoToolbox EveCanvas_CFLAGS = -fobjc-arc -Wall -Wextra -DGLES_SILENCE_DEPRECATION EveCanvas_CODESIGN_FLAGS = -Sentitlements.plist EveCanvas_INSTALL_PATH = /Applications diff --git a/Sources/EVEFrameStreamClient.h b/Sources/EVEFrameStreamClient.h index 16f8fa8..9c0860b 100644 --- a/Sources/EVEFrameStreamClient.h +++ b/Sources/EVEFrameStreamClient.h @@ -5,6 +5,7 @@ @interface EVEFrameStreamClient : NSObject - (instancetype)initWithURL:(NSURL *)url delegate:(id)delegate; +- (instancetype)initWithURLs:(NSArray *)urls delegate:(id)delegate; - (void)connect; - (void)disconnect; - (void)sendPointerPhase:(NSString *)phase x:(CGFloat)x y:(CGFloat)y; @@ -14,7 +15,9 @@ @protocol EVEFrameStreamClientDelegate - (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveImage:(UIImage *)image; +- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveVideoAccessUnit:(NSData *)data; - (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveViewportWidth:(CGFloat)width height:(CGFloat)height scale:(CGFloat)scale; +- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveCodec:(NSString *)codec; - (void)frameStreamClient:(EVEFrameStreamClient *)client didChangeStatus:(NSString *)status; @end diff --git a/Sources/EVEFrameStreamClient.m b/Sources/EVEFrameStreamClient.m index 4799ecf..89ec22e 100644 --- a/Sources/EVEFrameStreamClient.m +++ b/Sources/EVEFrameStreamClient.m @@ -2,7 +2,8 @@ @interface EVEFrameStreamClient () -@property(nonatomic, strong) NSURL *url; +@property(nonatomic, copy) NSArray *urls; +@property(nonatomic, assign) NSUInteger urlIndex; @property(nonatomic, weak) id delegate; @property(nonatomic, strong) NSURLSession *session; @property(nonatomic, strong) NSURLSessionWebSocketTask *task; @@ -13,12 +14,16 @@ @implementation EVEFrameStreamClient - (instancetype)initWithURL:(NSURL *)url delegate:(id)delegate { + return [self initWithURLs:url ? @[url] : @[] delegate:delegate]; +} + +- (instancetype)initWithURLs:(NSArray *)urls delegate:(id)delegate { self = [super init]; if (!self) { return nil; } - self.url = url; + self.urls = urls.count > 0 ? urls : @[]; self.delegate = delegate; self.session = [NSURLSession sessionWithConfiguration:NSURLSessionConfiguration.defaultSessionConfiguration]; self.shouldReconnect = YES; @@ -28,9 +33,15 @@ - (void)connect { self.shouldReconnect = YES; [self.task cancelWithCloseCode:NSURLSessionWebSocketCloseCodeNormalClosure reason:nil]; - self.task = [self.session webSocketTaskWithURL:self.url]; + NSURL *url = self.urls.count > 0 ? self.urls[self.urlIndex % self.urls.count] : nil; + if (!url) { + [self publishStatus:@"stream URL missing"]; + return; + } + + self.task = [self.session webSocketTaskWithURL:url]; [self.task resume]; - [self publishStatus:@"connecting"]; + [self publishStatus:[NSString stringWithFormat:@"connecting %@", url.host ?: @""]]; [self receiveNextMessage]; } @@ -80,10 +91,18 @@ } if (message.type == NSURLSessionWebSocketMessageTypeData) { - UIImage *image = [UIImage imageWithData:message.data]; - if (image) { + const uint8_t *bytes = message.data.bytes; + BOOL isJpeg = message.data.length > 2 && bytes[0] == 0xff && bytes[1] == 0xd8; + if (isJpeg) { + UIImage *image = [UIImage imageWithData:message.data]; + if (image) { + dispatch_async(dispatch_get_main_queue(), ^{ + [self.delegate frameStreamClient:self didReceiveImage:image]; + }); + } + } else { dispatch_async(dispatch_get_main_queue(), ^{ - [self.delegate frameStreamClient:self didReceiveImage:image]; + [self.delegate frameStreamClient:self didReceiveVideoAccessUnit:message.data]; }); } } else if (message.type == NSURLSessionWebSocketMessageTypeString) { @@ -114,6 +133,10 @@ } dispatch_async(dispatch_get_main_queue(), ^{ [self.delegate frameStreamClient:self didReceiveViewportWidth:width height:height scale:scale]; + NSString *codec = message[@"codec"]; + if ([codec isKindOfClass:NSString.class]) { + [self.delegate frameStreamClient:self didReceiveCodec:codec]; + } [self.delegate frameStreamClient:self didChangeStatus:@"stream live"]; }); } @@ -124,6 +147,10 @@ return; } + if (self.urls.count > 1) { + self.urlIndex = (self.urlIndex + 1) % self.urls.count; + } + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(2.0 * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{ if (self.shouldReconnect) { [self connect]; diff --git a/Sources/EVEH264StreamDecoder.h b/Sources/EVEH264StreamDecoder.h new file mode 100644 index 0000000..015469d --- /dev/null +++ b/Sources/EVEH264StreamDecoder.h @@ -0,0 +1,15 @@ +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface EVEH264StreamDecoder : NSObject + +@property(nonatomic, readonly) AVSampleBufferDisplayLayer *displayLayer; + +- (void)reset; +- (void)consumeAnnexBAccessUnit:(NSData *)data; + +@end + +NS_ASSUME_NONNULL_END diff --git a/Sources/EVEH264StreamDecoder.m b/Sources/EVEH264StreamDecoder.m new file mode 100644 index 0000000..82f4eb0 --- /dev/null +++ b/Sources/EVEH264StreamDecoder.m @@ -0,0 +1,205 @@ +#import "EVEH264StreamDecoder.h" + +#import +#import +#import + +@interface EVEH264StreamDecoder () + +@property(nonatomic, strong) AVSampleBufferDisplayLayer *displayLayer; +@property(nonatomic) CMVideoFormatDescriptionRef formatDescription; +@property(nonatomic, strong) NSData *sps; +@property(nonatomic, strong) NSData *pps; +@property(nonatomic, assign) int64_t frameIndex; + +@end + +@implementation EVEH264StreamDecoder + +- (instancetype)init { + self = [super init]; + if (!self) { + return nil; + } + + _displayLayer = [[AVSampleBufferDisplayLayer alloc] init]; + _displayLayer.videoGravity = AVLayerVideoGravityResizeAspect; + _displayLayer.backgroundColor = UIColor.blackColor.CGColor; + return self; +} + +- (void)dealloc { + [self reset]; +} + +- (void)reset { + if (_formatDescription) { + CFRelease(_formatDescription); + _formatDescription = NULL; + } + self.sps = nil; + self.pps = nil; + self.frameIndex = 0; + [self.displayLayer flushAndRemoveImage]; +} + +- (void)consumeAnnexBAccessUnit:(NSData *)data { + NSArray *nals = [self parseAnnexB:data]; + if (nals.count == 0) { + return; + } + + NSMutableData *avcc = [NSMutableData data]; + BOOL hasSlice = NO; + for (NSData *nal in nals) { + if (nal.length == 0) { + continue; + } + + const uint8_t *bytes = nal.bytes; + uint8_t nalType = bytes[0] & 0x1f; + if (nalType == 7) { + self.sps = nal; + [self rebuildFormatDescriptionIfReady]; + continue; + } + if (nalType == 8) { + self.pps = nal; + [self rebuildFormatDescriptionIfReady]; + continue; + } + if (nalType == 1 || nalType == 5) { + hasSlice = YES; + } + + uint32_t length = CFSwapInt32HostToBig((uint32_t)nal.length); + [avcc appendBytes:&length length:sizeof(length)]; + [avcc appendData:nal]; + } + + if (!hasSlice || !_formatDescription || avcc.length == 0) { + return; + } + + CMBlockBufferRef blockBuffer = NULL; + OSStatus status = CMBlockBufferCreateWithMemoryBlock( + kCFAllocatorDefault, + NULL, + avcc.length, + kCFAllocatorDefault, + NULL, + 0, + avcc.length, + 0, + &blockBuffer); + if (status != noErr || !blockBuffer) { + return; + } + + status = CMBlockBufferReplaceDataBytes(avcc.bytes, blockBuffer, 0, avcc.length); + if (status != noErr) { + CFRelease(blockBuffer); + return; + } + + CMSampleTimingInfo timing = { + .duration = CMTimeMake(1, 60), + .presentationTimeStamp = CMTimeMake(self.frameIndex++, 60), + .decodeTimeStamp = kCMTimeInvalid, + }; + size_t sampleSize = avcc.length; + CMSampleBufferRef sampleBuffer = NULL; + status = CMSampleBufferCreateReady( + kCFAllocatorDefault, + blockBuffer, + _formatDescription, + 1, + 1, + &timing, + 1, + &sampleSize, + &sampleBuffer); + CFRelease(blockBuffer); + if (status != noErr || !sampleBuffer) { + return; + } + + CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, YES); + if (attachments && CFArrayGetCount(attachments) > 0) { + CFMutableDictionaryRef attachment = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachments, 0); + CFDictionarySetValue(attachment, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue); + } + + if (self.displayLayer.status == AVQueuedSampleBufferRenderingStatusFailed) { + [self.displayLayer flush]; + } + [self.displayLayer enqueueSampleBuffer:sampleBuffer]; + CFRelease(sampleBuffer); +} + +- (void)rebuildFormatDescriptionIfReady { + if (!self.sps || !self.pps) { + return; + } + + if (_formatDescription) { + CFRelease(_formatDescription); + _formatDescription = NULL; + } + + const uint8_t *parameterSets[] = { self.sps.bytes, self.pps.bytes }; + size_t parameterSetSizes[] = { self.sps.length, self.pps.length }; + CMVideoFormatDescriptionCreateFromH264ParameterSets( + kCFAllocatorDefault, + 2, + parameterSets, + parameterSetSizes, + 4, + &_formatDescription); +} + +- (NSArray *)parseAnnexB:(NSData *)data { + NSMutableArray *nals = [NSMutableArray array]; + const uint8_t *bytes = data.bytes; + NSUInteger length = data.length; + NSUInteger cursor = 0; + while (cursor + 3 < length) { + NSUInteger start = [self findStartCode:bytes length:length offset:cursor startCodeLength:NULL]; + if (start == NSNotFound) { + break; + } + + NSUInteger startCodeLength = 0; + [self findStartCode:bytes length:length offset:start startCodeLength:&startCodeLength]; + NSUInteger payloadStart = start + startCodeLength; + NSUInteger next = [self findStartCode:bytes length:length offset:payloadStart startCodeLength:NULL]; + NSUInteger payloadEnd = next == NSNotFound ? length : next; + if (payloadEnd > payloadStart) { + [nals addObject:[data subdataWithRange:NSMakeRange(payloadStart, payloadEnd - payloadStart)]]; + } + cursor = payloadEnd; + } + + return nals; +} + +- (NSUInteger)findStartCode:(const uint8_t *)bytes length:(NSUInteger)length offset:(NSUInteger)offset startCodeLength:(NSUInteger *)startCodeLength { + for (NSUInteger index = offset; index + 3 < length; index++) { + if (bytes[index] == 0 && bytes[index + 1] == 0 && bytes[index + 2] == 1) { + if (startCodeLength) { + *startCodeLength = 3; + } + return index; + } + if (index + 4 < length && bytes[index] == 0 && bytes[index + 1] == 0 && bytes[index + 2] == 0 && bytes[index + 3] == 1) { + if (startCodeLength) { + *startCodeLength = 4; + } + return index; + } + } + + return NSNotFound; +} + +@end diff --git a/Sources/EVEViewController.mm b/Sources/EVEViewController.mm index a070dae..67ddd7f 100644 --- a/Sources/EVEViewController.mm +++ b/Sources/EVEViewController.mm @@ -5,11 +5,13 @@ #import "EVEFrameStreamClient.h" #import "EVEGLView.h" +#import "EVEH264StreamDecoder.h" @interface EVEViewController () @property(nonatomic, strong) EVEGLView *glView; @property(nonatomic, strong) UIImageView *streamImageView; +@property(nonatomic, strong) EVEH264StreamDecoder *videoDecoder; @property(nonatomic, strong) UILabel *overlayLabel; @property(nonatomic, strong) CADisplayLink *displayLink; @property(nonatomic, strong) CMMotionManager *motionManager; @@ -20,6 +22,7 @@ @property(nonatomic, assign) CGSize streamViewportSize; @property(nonatomic, assign) CGFloat streamScale; @property(nonatomic, copy) NSString *streamStatus; +@property(nonatomic, copy) NSString *streamCodec; @end @@ -41,6 +44,11 @@ self.streamImageView.userInteractionEnabled = NO; [root addSubview:self.streamImageView]; + self.videoDecoder = [[EVEH264StreamDecoder alloc] init]; + self.videoDecoder.displayLayer.frame = root.bounds; + self.videoDecoder.displayLayer.hidden = YES; + [root.layer addSublayer:self.videoDecoder.displayLayer]; + self.overlayLabel = [[UILabel alloc] initWithFrame:CGRectZero]; self.overlayLabel.translatesAutoresizingMaskIntoConstraints = NO; self.overlayLabel.numberOfLines = 0; @@ -77,8 +85,12 @@ self.streamViewportSize = CGSizeMake(1620.0, 2160.0); self.streamScale = 2.0; self.streamStatus = @"stream idle"; - NSURL *streamURL = [NSURL URLWithString:@"ws://192.168.1.66:8792/stream"]; - self.streamClient = [[EVEFrameStreamClient alloc] initWithURL:streamURL delegate:self]; + self.streamCodec = @"jpeg"; + NSArray *streamURLs = @[ + [NSURL URLWithString:@"ws://127.0.0.1:8792/stream"], + [NSURL URLWithString:@"ws://192.168.1.66:8792/stream"], + ]; + self.streamClient = [[EVEFrameStreamClient alloc] initWithURLs:streamURLs delegate:self]; [self.streamClient connect]; self.displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(frameTick:)]; @@ -107,6 +119,7 @@ - (void)viewDidLayoutSubviews { [super viewDidLayoutSubviews]; [self.glView resizeDrawableIfNeeded]; + self.videoDecoder.displayLayer.frame = self.view.bounds; } - (void)frameTick:(CADisplayLink *)link { @@ -136,12 +149,14 @@ @"EVE Canvas\n" "CEF stream + native touch\n" "%@\n" + "codec %@\n" "points %.0fx%.0f pixels %.0fx%.0f @ %.1fx\n" "stream %.0fx%.0f @ %.1fx\n" "fps %.1f touches %lu\n" "accel %+0.2f %+0.2f %+0.2f\n" "gyro %+0.2f %+0.2f %+0.2f", self.streamStatus ?: @"stream", + self.streamCodec ?: @"unknown", points.width, points.height, pixels.width, pixels.height, scale, self.streamViewportSize.width, self.streamViewportSize.height, self.streamScale, self.filteredFPS, (unsigned long)self.touchCount, @@ -211,9 +226,18 @@ - (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveImage:(UIImage *)image { (void)client; + self.videoDecoder.displayLayer.hidden = YES; + self.streamImageView.hidden = NO; self.streamImageView.image = image; } +- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveVideoAccessUnit:(NSData *)data { + (void)client; + self.streamImageView.hidden = YES; + self.videoDecoder.displayLayer.hidden = NO; + [self.videoDecoder consumeAnnexBAccessUnit:data]; +} + - (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveViewportWidth:(CGFloat)width height:(CGFloat)height scale:(CGFloat)scale { (void)client; if (width > 0 && height > 0) { @@ -224,6 +248,14 @@ } } +- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveCodec:(NSString *)codec { + (void)client; + if (![codec isEqualToString:self.streamCodec]) { + [self.videoDecoder reset]; + } + self.streamCodec = codec; +} + - (void)frameStreamClient:(EVEFrameStreamClient *)client didChangeStatus:(NSString *)status { (void)client; self.streamStatus = status;