Flutter iOS Embedder
FlutterMetalLayer.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
6 
7 #include <CoreMedia/CoreMedia.h>
8 #include <IOSurface/IOSurfaceObjC.h>
9 #include <Metal/Metal.h>
10 #include <UIKit/UIKit.h>
11 
12 #import "flutter/shell/platform/darwin/common/InternalFlutterSwiftCommon/InternalFlutterSwiftCommon.h"
14 
16 
17 @interface DisplayLinkManager : NSObject
18 @property(class, nonatomic, readonly) BOOL maxRefreshRateEnabledOnIPhone;
19 + (double)displayRefreshRate;
20 @end
21 
22 @class FlutterTexture;
23 @class FlutterDrawable;
24 
25 extern CFTimeInterval display_link_target;
26 
27 @interface FlutterMetalLayer () {
28  id<MTLDevice> _preferredDevice;
29  CGSize _drawableSize;
30 
31  NSUInteger _nextDrawableId;
32 
33  NSMutableSet<FlutterTexture*>* _availableTextures;
34  NSUInteger _totalTextures;
35 
37 
38  // There must be a CADisplayLink scheduled *on main thread* otherwise
39  // core animation only updates layers 60 times a second.
40  CADisplayLink* _displayLink;
42 
43  // Used to track whether the content was set during this display link.
44  // When unlocking phone the layer (main thread) display link and raster thread
45  // display link get out of sync for several seconds. Even worse, layer display
46  // link does not seem to reflect actual vsync. Forcing the layer link
47  // to max rate (instead range) temporarily seems to fix the issue.
49 
50  // Whether layer displayLink is forced to max rate.
52 }
53 
54 - (void)onDisplayLink:(CADisplayLink*)link;
55 - (void)presentTexture:(FlutterTexture*)texture;
56 - (void)returnTexture:(FlutterTexture*)texture;
57 
58 @end
59 
60 @interface FlutterTexture : NSObject
61 
62 @property(readonly, nonatomic) id<MTLTexture> texture;
63 @property(readonly, nonatomic) IOSurface* surface;
64 @property(readwrite, nonatomic) CFTimeInterval presentedTime;
65 @property(readwrite, atomic) BOOL waitingForCompletion;
66 
67 @end
68 
69 @implementation FlutterTexture
70 
71 - (instancetype)initWithTexture:(id<MTLTexture>)texture surface:(IOSurface*)surface {
72  if (self = [super init]) {
73  _texture = texture;
74  _surface = surface;
75  }
76  return self;
77 }
78 
79 @end
80 
81 @interface FlutterDrawable : NSObject <FlutterMetalDrawable> {
84  NSUInteger _drawableId;
85  BOOL _presented;
86 }
87 
88 - (instancetype)initWithTexture:(FlutterTexture*)texture
89  layer:(FlutterMetalLayer*)layer
90  drawableId:(NSUInteger)drawableId;
91 
92 @end
93 
94 @implementation FlutterDrawable
95 
96 - (instancetype)initWithTexture:(FlutterTexture*)texture
97  layer:(FlutterMetalLayer*)layer
98  drawableId:(NSUInteger)drawableId {
99  if (self = [super init]) {
100  _texture = texture;
101  _layer = layer;
102  _drawableId = drawableId;
103  }
104  return self;
105 }
106 
107 - (id<MTLTexture>)texture {
108  return self->_texture.texture;
109 }
110 
111 #pragma clang diagnostic push
112 #pragma clang diagnostic ignored "-Wunguarded-availability-new"
113 - (CAMetalLayer*)layer {
114  return (id)self->_layer;
115 }
116 #pragma clang diagnostic pop
117 
118 - (NSUInteger)drawableID {
119  return self->_drawableId;
120 }
121 
122 - (CFTimeInterval)presentedTime {
123  return 0;
124 }
125 
126 - (void)present {
127  [_layer presentTexture:self->_texture];
128  self->_presented = YES;
129 }
130 
131 - (void)dealloc {
132  if (!_presented) {
133  [_layer returnTexture:self->_texture];
134  }
135 }
136 
137 - (void)addPresentedHandler:(nonnull MTLDrawablePresentedHandler)block {
138  [FlutterLogger logWarning:@"FlutterMetalLayer drawable does not implement addPresentedHandler:"];
139 }
140 
141 - (void)presentAtTime:(CFTimeInterval)presentationTime {
142  [FlutterLogger logWarning:@"FlutterMetalLayer drawable does not implement presentAtTime:"];
143 }
144 
145 - (void)presentAfterMinimumDuration:(CFTimeInterval)duration {
146  [FlutterLogger
147  logWarning:@"FlutterMetalLayer drawable does not implement presentAfterMinimumDuration:"];
148 }
149 
150 - (void)flutterPrepareForPresent:(nonnull id<MTLCommandBuffer>)commandBuffer {
151  FlutterTexture* texture = _texture;
152  texture.waitingForCompletion = YES;
153  [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> buffer) {
154  texture.waitingForCompletion = NO;
155  }];
156 }
157 
158 @end
159 
160 @interface FlutterMetalLayerDisplayLinkProxy : NSObject {
162 }
163 
164 @end
165 
166 @implementation FlutterMetalLayerDisplayLinkProxy
167 - (instancetype)initWithLayer:(FlutterMetalLayer*)layer {
168  if (self = [super init]) {
169  _layer = layer;
170  }
171  return self;
172 }
173 
174 - (void)onDisplayLink:(CADisplayLink*)link {
175  [_layer onDisplayLink:link];
176 }
177 
178 @end
179 
180 @implementation FlutterMetalLayer
181 
182 - (instancetype)init {
183  if (self = [super init]) {
184  _preferredDevice = MTLCreateSystemDefaultDevice();
185  self.device = self.preferredDevice;
186  self.pixelFormat = MTLPixelFormatBGRA8Unorm;
187  _availableTextures = [[NSMutableSet alloc] init];
188 
190  [[FlutterMetalLayerDisplayLinkProxy alloc] initWithLayer:self];
191  _displayLink = [CADisplayLink displayLinkWithTarget:proxy selector:@selector(onDisplayLink:)];
192  [self setMaxRefreshRate:DisplayLinkManager.displayRefreshRate forceMax:NO];
193  [_displayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSRunLoopCommonModes];
194  [[NSNotificationCenter defaultCenter] addObserver:self
195  selector:@selector(didEnterBackground:)
196  name:UIApplicationDidEnterBackgroundNotification
197  object:nil];
198  }
199  return self;
200 }
201 
202 - (void)dealloc {
203  [_displayLink invalidate];
204  [[NSNotificationCenter defaultCenter] removeObserver:self];
205 }
206 
207 - (void)setMaxRefreshRate:(double)refreshRate forceMax:(BOOL)forceMax {
208  // This is copied from vsync_waiter_ios.mm. The vsync waiter has display link scheduled on UI
209  // thread which does not trigger actual core animation frame. As a workaround FlutterMetalLayer
210  // has it's own displaylink scheduled on main thread, which is used to trigger core animation
211  // frame allowing for 120hz updates.
213  return;
214  }
215  double maxFrameRate = fmax(refreshRate, 60);
216  double minFrameRate = fmax(maxFrameRate / 2, 60);
217  if (@available(iOS 15.0, *)) {
218  _displayLink.preferredFrameRateRange =
219  CAFrameRateRangeMake(forceMax ? maxFrameRate : minFrameRate, maxFrameRate, maxFrameRate);
220  } else {
221  _displayLink.preferredFramesPerSecond = maxFrameRate;
222  }
223 }
224 
225 - (void)onDisplayLink:(CADisplayLink*)link {
226  _didSetContentsDuringThisDisplayLinkPeriod = NO;
227  // Do not pause immediately, this seems to prevent 120hz while touching.
228  if (_displayLinkPauseCountdown == 3) {
229  _displayLink.paused = YES;
230  if (_displayLinkForcedMaxRate) {
231  [self setMaxRefreshRate:DisplayLinkManager.displayRefreshRate forceMax:NO];
232  _displayLinkForcedMaxRate = NO;
233  }
234  } else {
235  ++_displayLinkPauseCountdown;
236  }
237 }
238 
239 - (BOOL)isKindOfClass:(Class)aClass {
240 #pragma clang diagnostic push
241 #pragma clang diagnostic ignored "-Wunguarded-availability-new"
242  // Pretend that we're a CAMetalLayer so that the rest of Flutter plays along
243  if ([aClass isEqual:[CAMetalLayer class]]) {
244  return YES;
245  }
246 #pragma clang diagnostic pop
247  return [super isKindOfClass:aClass];
248 }
249 
250 - (void)setDrawableSize:(CGSize)drawableSize {
251  [_availableTextures removeAllObjects];
252  _front = nil;
253  _totalTextures = 0;
254  _drawableSize = drawableSize;
255 }
256 
257 - (void)didEnterBackground:(id)notification {
258  [_availableTextures removeAllObjects];
259  _totalTextures = _front != nil ? 1 : 0;
260  _displayLink.paused = YES;
261 }
262 
263 - (CGSize)drawableSize {
264  return _drawableSize;
265 }
266 
267 - (IOSurface*)createIOSurface {
268  unsigned pixelFormat;
269  unsigned bytesPerElement;
270  if (self.pixelFormat == MTLPixelFormatRGBA16Float) {
271  pixelFormat = kCVPixelFormatType_64RGBAHalf;
272  bytesPerElement = 8;
273  } else if (self.pixelFormat == MTLPixelFormatBGRA8Unorm) {
274  pixelFormat = kCVPixelFormatType_32BGRA;
275  bytesPerElement = 4;
276  } else if (self.pixelFormat == MTLPixelFormatBGRA10_XR) {
277  pixelFormat = kCVPixelFormatType_40ARGBLEWideGamut;
278  bytesPerElement = 8;
279  } else {
280  NSString* errorMessage =
281  [NSString stringWithFormat:@"Unsupported pixel format: %lu", self.pixelFormat];
282  [FlutterLogger logError:errorMessage];
283  return nil;
284  }
285  size_t bytesPerRow =
286  IOSurfaceAlignProperty(kIOSurfaceBytesPerRow, _drawableSize.width * bytesPerElement);
287  size_t totalBytes =
288  IOSurfaceAlignProperty(kIOSurfaceAllocSize, _drawableSize.height * bytesPerRow);
289  NSDictionary* options = @{
290  (id)kIOSurfaceWidth : @(_drawableSize.width),
291  (id)kIOSurfaceHeight : @(_drawableSize.height),
292  (id)kIOSurfacePixelFormat : @(pixelFormat),
293  (id)kIOSurfaceBytesPerElement : @(bytesPerElement),
294  (id)kIOSurfaceBytesPerRow : @(bytesPerRow),
295  (id)kIOSurfaceAllocSize : @(totalBytes),
296  };
297 
298  IOSurfaceRef res = IOSurfaceCreate((CFDictionaryRef)options);
299  if (res == nil) {
300  NSString* errorMessage = [NSString
301  stringWithFormat:@"Failed to create IOSurface with options %@", options.debugDescription];
302  [FlutterLogger logError:errorMessage];
303  return nil;
304  }
305 
306  if (self.colorspace != nil) {
307  CFStringRef name = CGColorSpaceGetName(self.colorspace);
308  IOSurfaceSetValue(res, kIOSurfaceColorSpace, name);
309  } else {
310  IOSurfaceSetValue(res, kIOSurfaceColorSpace, kCGColorSpaceSRGB);
311  }
312  return (__bridge_transfer IOSurface*)res;
313 }
314 
315 - (FlutterTexture*)nextTexture {
316  CFTimeInterval start = CACurrentMediaTime();
317  while (true) {
318  FlutterTexture* texture = [self tryNextTexture];
319  if (texture != nil) {
320  return texture;
321  }
322  CFTimeInterval elapsed = CACurrentMediaTime() - start;
323  if (elapsed > 1.0) {
324  NSLog(@"Waited %f seconds for a drawable, giving up.", elapsed);
325  return nil;
326  }
327  }
328 }
329 
330 - (FlutterTexture*)tryNextTexture {
331  @synchronized(self) {
332  if (_front != nil && _front.waitingForCompletion) {
333  return nil;
334  }
335  if (_totalTextures < 3) {
336  ++_totalTextures;
337  IOSurface* surface = [self createIOSurface];
338  if (surface == nil) {
339  return nil;
340  }
341  MTLTextureDescriptor* textureDescriptor =
342  [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:_pixelFormat
343  width:_drawableSize.width
344  height:_drawableSize.height
345  mipmapped:NO];
346 
347  if (_framebufferOnly) {
348  textureDescriptor.usage = MTLTextureUsageRenderTarget;
349  } else {
350  textureDescriptor.usage =
351  MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite;
352  }
353  id<MTLTexture> texture = [self.device newTextureWithDescriptor:textureDescriptor
354  iosurface:(__bridge IOSurfaceRef)surface
355  plane:0];
356  FlutterTexture* flutterTexture = [[FlutterTexture alloc] initWithTexture:texture
357  surface:surface];
358  return flutterTexture;
359  } else {
360  // Prefer surface that is not in use and has been presented the longest
361  // time ago.
362  // When isInUse is false, the surface is definitely not used by the compositor.
363  // When isInUse is true, the surface may be used by the compositor.
364  // When both surfaces are in use, the one presented earlier will be returned.
365  // The assumption here is that the compositor is already aware of the
366  // newer texture and is unlikely to read from the older one, even though it
367  // has not decreased the use count yet (there seems to be certain latency).
368  FlutterTexture* res = nil;
369  for (FlutterTexture* texture in _availableTextures) {
370  if (res == nil) {
371  res = texture;
372  } else if (res.surface.isInUse && !texture.surface.isInUse) {
373  // prefer texture that is not in use.
374  res = texture;
375  } else if (res.surface.isInUse == texture.surface.isInUse &&
376  texture.presentedTime < res.presentedTime) {
377  // prefer texture with older presented time.
378  res = texture;
379  }
380  }
381  if (res != nil) {
382  [_availableTextures removeObject:res];
383  }
384  return res;
385  }
386  }
387 }
388 
389 - (id<CAMetalDrawable>)nextDrawable {
390  FlutterTexture* texture = [self nextTexture];
391  if (texture == nil) {
392  return nil;
393  }
394  FlutterDrawable* drawable = [[FlutterDrawable alloc] initWithTexture:texture
395  layer:self
396  drawableId:_nextDrawableId++];
397  return drawable;
398 }
399 
400 - (void)presentOnMainThread:(FlutterTexture*)texture {
401  // This is needed otherwise frame gets skipped on touch begin / end. Go figure.
402  // Might also be placebo
403  [self setNeedsDisplay];
404 
405  [CATransaction begin];
406  [CATransaction setDisableActions:YES];
407  self.contents = texture.surface;
408  [CATransaction commit];
409  _displayLink.paused = NO;
410  _displayLinkPauseCountdown = 0;
411  if (!_didSetContentsDuringThisDisplayLinkPeriod) {
412  _didSetContentsDuringThisDisplayLinkPeriod = YES;
413  } else if (!_displayLinkForcedMaxRate) {
414  _displayLinkForcedMaxRate = YES;
415  [self setMaxRefreshRate:DisplayLinkManager.displayRefreshRate forceMax:YES];
416  }
417 }
418 
419 - (void)presentTexture:(FlutterTexture*)texture {
420  @synchronized(self) {
421  if (_front != nil) {
422  [_availableTextures addObject:_front];
423  }
424  _front = texture;
425  texture.presentedTime = CACurrentMediaTime();
426  if ([NSThread isMainThread]) {
427  [self presentOnMainThread:texture];
428  } else {
429  // Core animation layers can only be updated on main thread.
430  dispatch_async(dispatch_get_main_queue(), ^{
431  [self presentOnMainThread:texture];
432  });
433  }
434  }
435 }
436 
437 - (void)returnTexture:(FlutterTexture*)texture {
438  @synchronized(self) {
439  [_availableTextures addObject:texture];
440  }
441 }
442 
443 + (BOOL)enabled {
444  static BOOL enabled = YES;
445  static BOOL didCheckInfoPlist = NO;
446  if (!didCheckInfoPlist) {
447  didCheckInfoPlist = YES;
448  NSNumber* use_flutter_metal_layer =
449  [[NSBundle mainBundle] objectForInfoDictionaryKey:@"FLTUseFlutterMetalLayer"];
450  if (use_flutter_metal_layer != nil && ![use_flutter_metal_layer boolValue]) {
451  enabled = NO;
452  }
453  }
454  return enabled;
455 }
456 
457 @end
CFTimeInterval display_link_target
id< MTLDevice > _preferredDevice
NSMutableSet< FlutterTexture * > * _availableTextures
BOOL _didSetContentsDuringThisDisplayLinkPeriod
CADisplayLink * _displayLink
FlutterTexture * _front
NSUInteger _displayLinkPauseCountdown
FlutterTexture * _texture
NSUInteger _drawableId
__weak FlutterMetalLayer * _layer
CGColorSpaceRef colorspace
nullable id< CAMetalDrawable > nextDrawable()
MTLPixelFormat pixelFormat
IOSurface * surface
id< MTLTexture > texture
CFTimeInterval presentedTime
CADisplayLink * _displayLink