Add native H264 EveCanvas stream decoder

Makefile

@@ -10,9 +10,10 @@ EveCanvas_FILES = \
 	Sources/EVEAppDelegate.m \
 	Sources/EVEViewController.mm \
 	Sources/EVEGLView.mm \
-	Sources/EVEFrameStreamClient.m
+	Sources/EVEFrameStreamClient.m \
+	Sources/EVEH264StreamDecoder.m
 
-EveCanvas_FRAMEWORKS = UIKit QuartzCore OpenGLES CoreMotion
+EveCanvas_FRAMEWORKS = UIKit QuartzCore OpenGLES CoreMotion AVFoundation CoreMedia CoreVideo VideoToolbox
 EveCanvas_CFLAGS = -fobjc-arc -Wall -Wextra -DGLES_SILENCE_DEPRECATION
 EveCanvas_CODESIGN_FLAGS = -Sentitlements.plist
 EveCanvas_INSTALL_PATH = /Applications

Sources/EVEFrameStreamClient.h

@@ -5,6 +5,7 @@
 @interface EVEFrameStreamClient : NSObject
 
 - (instancetype)initWithURL:(NSURL *)url delegate:(id<EVEFrameStreamClientDelegate>)delegate;
+- (instancetype)initWithURLs:(NSArray<NSURL *> *)urls delegate:(id<EVEFrameStreamClientDelegate>)delegate;
 - (void)connect;
 - (void)disconnect;
 - (void)sendPointerPhase:(NSString *)phase x:(CGFloat)x y:(CGFloat)y;
@@ -14,7 +15,9 @@
 @protocol EVEFrameStreamClientDelegate <NSObject>
 
 - (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveImage:(UIImage *)image;
+- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveVideoAccessUnit:(NSData *)data;
 - (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveViewportWidth:(CGFloat)width height:(CGFloat)height scale:(CGFloat)scale;
+- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveCodec:(NSString *)codec;
 - (void)frameStreamClient:(EVEFrameStreamClient *)client didChangeStatus:(NSString *)status;
 
 @end

Sources/EVEFrameStreamClient.m

@@ -2,7 +2,8 @@
 
 @interface EVEFrameStreamClient ()
 
-@property(nonatomic, strong) NSURL *url;
+@property(nonatomic, copy) NSArray<NSURL *> *urls;
+@property(nonatomic, assign) NSUInteger urlIndex;
 @property(nonatomic, weak) id<EVEFrameStreamClientDelegate> delegate;
 @property(nonatomic, strong) NSURLSession *session;
 @property(nonatomic, strong) NSURLSessionWebSocketTask *task;
@@ -13,12 +14,16 @@
 @implementation EVEFrameStreamClient
 
 - (instancetype)initWithURL:(NSURL *)url delegate:(id<EVEFrameStreamClientDelegate>)delegate {
+  return [self initWithURLs:url ? @[url] : @[] delegate:delegate];
+}
+
+- (instancetype)initWithURLs:(NSArray<NSURL *> *)urls delegate:(id<EVEFrameStreamClientDelegate>)delegate {
   self = [super init];
   if (!self) {
     return nil;
   }
 
-  self.url = url;
+  self.urls = urls.count > 0 ? urls : @[];
   self.delegate = delegate;
   self.session = [NSURLSession sessionWithConfiguration:NSURLSessionConfiguration.defaultSessionConfiguration];
   self.shouldReconnect = YES;
@@ -28,9 +33,15 @@
 - (void)connect {
   self.shouldReconnect = YES;
   [self.task cancelWithCloseCode:NSURLSessionWebSocketCloseCodeNormalClosure reason:nil];
-  self.task = [self.session webSocketTaskWithURL:self.url];
+  NSURL *url = self.urls.count > 0 ? self.urls[self.urlIndex % self.urls.count] : nil;
+  if (!url) {
+    [self publishStatus:@"stream URL missing"];
+    return;
+  }
+
+  self.task = [self.session webSocketTaskWithURL:url];
   [self.task resume];
-  [self publishStatus:@"connecting"];
+  [self publishStatus:[NSString stringWithFormat:@"connecting %@", url.host ?: @""]];
   [self receiveNextMessage];
 }
 
@@ -80,10 +91,18 @@
     }
 
     if (message.type == NSURLSessionWebSocketMessageTypeData) {
-      UIImage *image = [UIImage imageWithData:message.data];
+      const uint8_t *bytes = message.data.bytes;
-      if (image) {
+      BOOL isJpeg = message.data.length > 2 && bytes[0] == 0xff && bytes[1] == 0xd8;
+      if (isJpeg) {
+        UIImage *image = [UIImage imageWithData:message.data];
+        if (image) {
+          dispatch_async(dispatch_get_main_queue(), ^{
+            [self.delegate frameStreamClient:self didReceiveImage:image];
+          });
+        }
+      } else {
         dispatch_async(dispatch_get_main_queue(), ^{
-          [self.delegate frameStreamClient:self didReceiveImage:image];
+          [self.delegate frameStreamClient:self didReceiveVideoAccessUnit:message.data];
         });
       }
     } else if (message.type == NSURLSessionWebSocketMessageTypeString) {
@@ -114,6 +133,10 @@
     }
     dispatch_async(dispatch_get_main_queue(), ^{
       [self.delegate frameStreamClient:self didReceiveViewportWidth:width height:height scale:scale];
+      NSString *codec = message[@"codec"];
+      if ([codec isKindOfClass:NSString.class]) {
+        [self.delegate frameStreamClient:self didReceiveCodec:codec];
+      }
       [self.delegate frameStreamClient:self didChangeStatus:@"stream live"];
     });
   }
@@ -124,6 +147,10 @@
     return;
   }
 
+  if (self.urls.count > 1) {
+    self.urlIndex = (self.urlIndex + 1) % self.urls.count;
+  }
+
   dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(2.0 * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{
     if (self.shouldReconnect) {
       [self connect];

Sources/EVEH264StreamDecoder.h

@@ -0,0 +1,15 @@
+#import <AVFoundation/AVFoundation.h>
+#import <Foundation/Foundation.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface EVEH264StreamDecoder : NSObject
+
+@property(nonatomic, readonly) AVSampleBufferDisplayLayer *displayLayer;
+
+- (void)reset;
+- (void)consumeAnnexBAccessUnit:(NSData *)data;
+
+@end
+
+NS_ASSUME_NONNULL_END

Sources/EVEH264StreamDecoder.m

@@ -0,0 +1,205 @@
+#import "EVEH264StreamDecoder.h"
+
+#import <CoreMedia/CoreMedia.h>
+#import <UIKit/UIKit.h>
+#import <VideoToolbox/VideoToolbox.h>
+
+@interface EVEH264StreamDecoder ()
+
+@property(nonatomic, strong) AVSampleBufferDisplayLayer *displayLayer;
+@property(nonatomic) CMVideoFormatDescriptionRef formatDescription;
+@property(nonatomic, strong) NSData *sps;
+@property(nonatomic, strong) NSData *pps;
+@property(nonatomic, assign) int64_t frameIndex;
+
+@end
+
+@implementation EVEH264StreamDecoder
+
+- (instancetype)init {
+  self = [super init];
+  if (!self) {
+    return nil;
+  }
+
+  _displayLayer = [[AVSampleBufferDisplayLayer alloc] init];
+  _displayLayer.videoGravity = AVLayerVideoGravityResizeAspect;
+  _displayLayer.backgroundColor = UIColor.blackColor.CGColor;
+  return self;
+}
+
+- (void)dealloc {
+  [self reset];
+}
+
+- (void)reset {
+  if (_formatDescription) {
+    CFRelease(_formatDescription);
+    _formatDescription = NULL;
+  }
+  self.sps = nil;
+  self.pps = nil;
+  self.frameIndex = 0;
+  [self.displayLayer flushAndRemoveImage];
+}
+
+- (void)consumeAnnexBAccessUnit:(NSData *)data {
+  NSArray<NSData *> *nals = [self parseAnnexB:data];
+  if (nals.count == 0) {
+    return;
+  }
+
+  NSMutableData *avcc = [NSMutableData data];
+  BOOL hasSlice = NO;
+  for (NSData *nal in nals) {
+    if (nal.length == 0) {
+      continue;
+    }
+
+    const uint8_t *bytes = nal.bytes;
+    uint8_t nalType = bytes[0] & 0x1f;
+    if (nalType == 7) {
+      self.sps = nal;
+      [self rebuildFormatDescriptionIfReady];
+      continue;
+    }
+    if (nalType == 8) {
+      self.pps = nal;
+      [self rebuildFormatDescriptionIfReady];
+      continue;
+    }
+    if (nalType == 1 || nalType == 5) {
+      hasSlice = YES;
+    }
+
+    uint32_t length = CFSwapInt32HostToBig((uint32_t)nal.length);
+    [avcc appendBytes:&length length:sizeof(length)];
+    [avcc appendData:nal];
+  }
+
+  if (!hasSlice || !_formatDescription || avcc.length == 0) {
+    return;
+  }
+
+  CMBlockBufferRef blockBuffer = NULL;
+  OSStatus status = CMBlockBufferCreateWithMemoryBlock(
+      kCFAllocatorDefault,
+      NULL,
+      avcc.length,
+      kCFAllocatorDefault,
+      NULL,
+      0,
+      avcc.length,
+      0,
+      &blockBuffer);
+  if (status != noErr || !blockBuffer) {
+    return;
+  }
+
+  status = CMBlockBufferReplaceDataBytes(avcc.bytes, blockBuffer, 0, avcc.length);
+  if (status != noErr) {
+    CFRelease(blockBuffer);
+    return;
+  }
+
+  CMSampleTimingInfo timing = {
+      .duration = CMTimeMake(1, 60),
+      .presentationTimeStamp = CMTimeMake(self.frameIndex++, 60),
+      .decodeTimeStamp = kCMTimeInvalid,
+  };
+  size_t sampleSize = avcc.length;
+  CMSampleBufferRef sampleBuffer = NULL;
+  status = CMSampleBufferCreateReady(
+      kCFAllocatorDefault,
+      blockBuffer,
+      _formatDescription,
+      1,
+      1,
+      &timing,
+      1,
+      &sampleSize,
+      &sampleBuffer);
+  CFRelease(blockBuffer);
+  if (status != noErr || !sampleBuffer) {
+    return;
+  }
+
+  CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, YES);
+  if (attachments && CFArrayGetCount(attachments) > 0) {
+    CFMutableDictionaryRef attachment = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachments, 0);
+    CFDictionarySetValue(attachment, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue);
+  }
+
+  if (self.displayLayer.status == AVQueuedSampleBufferRenderingStatusFailed) {
+    [self.displayLayer flush];
+  }
+  [self.displayLayer enqueueSampleBuffer:sampleBuffer];
+  CFRelease(sampleBuffer);
+}
+
+- (void)rebuildFormatDescriptionIfReady {
+  if (!self.sps || !self.pps) {
+    return;
+  }
+
+  if (_formatDescription) {
+    CFRelease(_formatDescription);
+    _formatDescription = NULL;
+  }
+
+  const uint8_t *parameterSets[] = { self.sps.bytes, self.pps.bytes };
+  size_t parameterSetSizes[] = { self.sps.length, self.pps.length };
+  CMVideoFormatDescriptionCreateFromH264ParameterSets(
+      kCFAllocatorDefault,
+      2,
+      parameterSets,
+      parameterSetSizes,
+      4,
+      &_formatDescription);
+}
+
+- (NSArray<NSData *> *)parseAnnexB:(NSData *)data {
+  NSMutableArray<NSData *> *nals = [NSMutableArray array];
+  const uint8_t *bytes = data.bytes;
+  NSUInteger length = data.length;
+  NSUInteger cursor = 0;
+  while (cursor + 3 < length) {
+    NSUInteger start = [self findStartCode:bytes length:length offset:cursor startCodeLength:NULL];
+    if (start == NSNotFound) {
+      break;
+    }
+
+    NSUInteger startCodeLength = 0;
+    [self findStartCode:bytes length:length offset:start startCodeLength:&startCodeLength];
+    NSUInteger payloadStart = start + startCodeLength;
+    NSUInteger next = [self findStartCode:bytes length:length offset:payloadStart startCodeLength:NULL];
+    NSUInteger payloadEnd = next == NSNotFound ? length : next;
+    if (payloadEnd > payloadStart) {
+      [nals addObject:[data subdataWithRange:NSMakeRange(payloadStart, payloadEnd - payloadStart)]];
+    }
+    cursor = payloadEnd;
+  }
+
+  return nals;
+}
+
+- (NSUInteger)findStartCode:(const uint8_t *)bytes length:(NSUInteger)length offset:(NSUInteger)offset startCodeLength:(NSUInteger *)startCodeLength {
+  for (NSUInteger index = offset; index + 3 < length; index++) {
+    if (bytes[index] == 0 && bytes[index + 1] == 0 && bytes[index + 2] == 1) {
+      if (startCodeLength) {
+        *startCodeLength = 3;
+      }
+      return index;
+    }
+    if (index + 4 < length && bytes[index] == 0 && bytes[index + 1] == 0 && bytes[index + 2] == 0 && bytes[index + 3] == 1) {
+      if (startCodeLength) {
+        *startCodeLength = 4;
+      }
+      return index;
+    }
+  }
+
+  return NSNotFound;
+}
+
+@end

Sources/EVEViewController.mm

@@ -5,11 +5,13 @@
 
 #import "EVEFrameStreamClient.h"
 #import "EVEGLView.h"
+#import "EVEH264StreamDecoder.h"
 
 @interface EVEViewController () <EVEFrameStreamClientDelegate>
 
 @property(nonatomic, strong) EVEGLView *glView;
 @property(nonatomic, strong) UIImageView *streamImageView;
+@property(nonatomic, strong) EVEH264StreamDecoder *videoDecoder;
 @property(nonatomic, strong) UILabel *overlayLabel;
 @property(nonatomic, strong) CADisplayLink *displayLink;
 @property(nonatomic, strong) CMMotionManager *motionManager;
@@ -20,6 +22,7 @@
 @property(nonatomic, assign) CGSize streamViewportSize;
 @property(nonatomic, assign) CGFloat streamScale;
 @property(nonatomic, copy) NSString *streamStatus;
+@property(nonatomic, copy) NSString *streamCodec;
 
 @end
 
@@ -41,6 +44,11 @@
   self.streamImageView.userInteractionEnabled = NO;
   [root addSubview:self.streamImageView];
 
+  self.videoDecoder = [[EVEH264StreamDecoder alloc] init];
+  self.videoDecoder.displayLayer.frame = root.bounds;
+  self.videoDecoder.displayLayer.hidden = YES;
+  [root.layer addSublayer:self.videoDecoder.displayLayer];
+
   self.overlayLabel = [[UILabel alloc] initWithFrame:CGRectZero];
   self.overlayLabel.translatesAutoresizingMaskIntoConstraints = NO;
   self.overlayLabel.numberOfLines = 0;
@@ -77,8 +85,12 @@
   self.streamViewportSize = CGSizeMake(1620.0, 2160.0);
   self.streamScale = 2.0;
   self.streamStatus = @"stream idle";
-  NSURL *streamURL = [NSURL URLWithString:@"ws://192.168.1.66:8792/stream"];
+  self.streamCodec = @"jpeg";
-  self.streamClient = [[EVEFrameStreamClient alloc] initWithURL:streamURL delegate:self];
+  NSArray<NSURL *> *streamURLs = @[
+    [NSURL URLWithString:@"ws://127.0.0.1:8792/stream"],
+    [NSURL URLWithString:@"ws://192.168.1.66:8792/stream"],
+  ];
+  self.streamClient = [[EVEFrameStreamClient alloc] initWithURLs:streamURLs delegate:self];
   [self.streamClient connect];
 
   self.displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(frameTick:)];
@@ -107,6 +119,7 @@
 - (void)viewDidLayoutSubviews {
   [super viewDidLayoutSubviews];
   [self.glView resizeDrawableIfNeeded];
+  self.videoDecoder.displayLayer.frame = self.view.bounds;
 }
 
 - (void)frameTick:(CADisplayLink *)link {
@@ -136,12 +149,14 @@
     @"EVE Canvas\n"
      "CEF stream + native touch\n"
      "%@\n"
+     "codec %@\n"
      "points %.0fx%.0f  pixels %.0fx%.0f @ %.1fx\n"
      "stream %.0fx%.0f @ %.1fx\n"
      "fps %.1f  touches %lu\n"
      "accel %+0.2f %+0.2f %+0.2f\n"
      "gyro  %+0.2f %+0.2f %+0.2f",
      self.streamStatus ?: @"stream",
+     self.streamCodec ?: @"unknown",
      points.width, points.height, pixels.width, pixels.height, scale,
      self.streamViewportSize.width, self.streamViewportSize.height, self.streamScale,
      self.filteredFPS, (unsigned long)self.touchCount,
@@ -211,9 +226,18 @@
 
 - (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveImage:(UIImage *)image {
   (void)client;
+  self.videoDecoder.displayLayer.hidden = YES;
+  self.streamImageView.hidden = NO;
   self.streamImageView.image = image;
 }
 
+- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveVideoAccessUnit:(NSData *)data {
+  (void)client;
+  self.streamImageView.hidden = YES;
+  self.videoDecoder.displayLayer.hidden = NO;
+  [self.videoDecoder consumeAnnexBAccessUnit:data];
+}
+
 - (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveViewportWidth:(CGFloat)width height:(CGFloat)height scale:(CGFloat)scale {
   (void)client;
   if (width > 0 && height > 0) {
@@ -224,6 +248,14 @@
   }
 }
 
+- (void)frameStreamClient:(EVEFrameStreamClient *)client didReceiveCodec:(NSString *)codec {
+  (void)client;
+  if (![codec isEqualToString:self.streamCodec]) {
+    [self.videoDecoder reset];
+  }
+  self.streamCodec = codec;
+}
+
 - (void)frameStreamClient:(EVEFrameStreamClient *)client didChangeStatus:(NSString *)status {
   (void)client;
   self.streamStatus = status;