Flutter iOS Embedder
FlutterMetalLayer.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
6 
7 #include <IOSurface/IOSurfaceObjC.h>
8 #include <Metal/Metal.h>
9 #include <UIKit/UIKit.h>
10 
11 #include "flutter/fml/logging.h"
13 
15 
16 @interface DisplayLinkManager : NSObject
17 @property(class, nonatomic, readonly) BOOL maxRefreshRateEnabledOnIPhone;
18 + (double)displayRefreshRate;
19 @end
20 
21 @class FlutterTexture;
22 @class FlutterDrawable;
23 
24 extern CFTimeInterval display_link_target;
25 
26 @interface FlutterMetalLayer () {
27  id<MTLDevice> _preferredDevice;
28  CGSize _drawableSize;
29 
30  NSUInteger _nextDrawableId;
31 
32  NSMutableSet<FlutterTexture*>* _availableTextures;
33  NSUInteger _totalTextures;
34 
36 
37  // There must be a CADisplayLink scheduled *on main thread* otherwise
38  // core animation only updates layers 60 times a second.
39  CADisplayLink* _displayLink;
41 
42  // Used to track whether the content was set during this display link.
43  // When unlocking phone the layer (main thread) display link and raster thread
44  // display link get out of sync for several seconds. Even worse, layer display
45  // link does not seem to reflect actual vsync. Forcing the layer link
46  // to max rate (instead range) temporarily seems to fix the issue.
48 
49  // Whether layer displayLink is forced to max rate.
51 }
52 
53 - (void)presentTexture:(FlutterTexture*)texture;
54 - (void)returnTexture:(FlutterTexture*)texture;
55 
56 @end
57 
58 @interface FlutterTexture : NSObject {
59  id<MTLTexture> _texture;
60  IOSurface* _surface;
61  CFTimeInterval _presentedTime;
62 }
63 
64 @property(readonly, nonatomic) id<MTLTexture> texture;
65 @property(readonly, nonatomic) IOSurface* surface;
66 @property(readwrite, nonatomic) CFTimeInterval presentedTime;
67 @property(readwrite, atomic) BOOL waitingForCompletion;
68 
69 @end
70 
71 @implementation FlutterTexture
72 
73 @synthesize texture = _texture;
74 @synthesize surface = _surface;
75 @synthesize presentedTime = _presentedTime;
76 @synthesize waitingForCompletion;
77 
78 - (instancetype)initWithTexture:(id<MTLTexture>)texture surface:(IOSurface*)surface {
79  if (self = [super init]) {
80  _texture = texture;
81  _surface = surface;
82  }
83  return self;
84 }
85 
86 @end
87 
88 @interface FlutterDrawable : NSObject <FlutterMetalDrawable> {
91  NSUInteger _drawableId;
92  BOOL _presented;
93 }
94 
95 - (instancetype)initWithTexture:(FlutterTexture*)texture
96  layer:(FlutterMetalLayer*)layer
97  drawableId:(NSUInteger)drawableId;
98 
99 @end
100 
101 @implementation FlutterDrawable
102 
103 - (instancetype)initWithTexture:(FlutterTexture*)texture
104  layer:(FlutterMetalLayer*)layer
105  drawableId:(NSUInteger)drawableId {
106  if (self = [super init]) {
107  _texture = texture;
108  _layer = layer;
109  _drawableId = drawableId;
110  }
111  return self;
112 }
113 
114 - (id<MTLTexture>)texture {
115  return self->_texture.texture;
116 }
117 
118 #pragma clang diagnostic push
119 #pragma clang diagnostic ignored "-Wunguarded-availability-new"
120 - (CAMetalLayer*)layer {
121  return (id)self->_layer;
122 }
123 #pragma clang diagnostic pop
124 
125 - (NSUInteger)drawableID {
126  return self->_drawableId;
127 }
128 
129 - (CFTimeInterval)presentedTime {
130  return 0;
131 }
132 
133 - (void)present {
134  [_layer presentTexture:self->_texture];
135  self->_presented = YES;
136 }
137 
138 - (void)dealloc {
139  if (!_presented) {
140  [_layer returnTexture:self->_texture];
141  }
142 }
143 
144 - (void)addPresentedHandler:(nonnull MTLDrawablePresentedHandler)block {
145  FML_LOG(WARNING) << "FlutterMetalLayer drawable does not implement addPresentedHandler:";
146 }
147 
148 - (void)presentAtTime:(CFTimeInterval)presentationTime {
149  FML_LOG(WARNING) << "FlutterMetalLayer drawable does not implement presentAtTime:";
150 }
151 
152 - (void)presentAfterMinimumDuration:(CFTimeInterval)duration {
153  FML_LOG(WARNING) << "FlutterMetalLayer drawable does not implement presentAfterMinimumDuration:";
154 }
155 
156 - (void)flutterPrepareForPresent:(nonnull id<MTLCommandBuffer>)commandBuffer {
157  FlutterTexture* texture = _texture;
158  texture.waitingForCompletion = YES;
159  [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> buffer) {
160  texture.waitingForCompletion = NO;
161  }];
162 }
163 
164 @end
165 
166 @implementation FlutterMetalLayer
167 
168 @synthesize preferredDevice = _preferredDevice;
169 @synthesize device = _device;
170 @synthesize pixelFormat = _pixelFormat;
171 @synthesize framebufferOnly = _framebufferOnly;
172 @synthesize colorspace = _colorspace;
173 @synthesize wantsExtendedDynamicRangeContent = _wantsExtendedDynamicRangeContent;
174 
175 - (instancetype)init {
176  if (self = [super init]) {
177  _preferredDevice = MTLCreateSystemDefaultDevice();
178  self.device = self.preferredDevice;
179  self.pixelFormat = MTLPixelFormatBGRA8Unorm;
180  _availableTextures = [[NSMutableSet alloc] init];
181 
182  _displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(onDisplayLink:)];
183  [self setMaxRefreshRate:DisplayLinkManager.displayRefreshRate forceMax:NO];
184  [_displayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSRunLoopCommonModes];
185  [[NSNotificationCenter defaultCenter] addObserver:self
186  selector:@selector(didEnterBackground:)
187  name:UIApplicationDidEnterBackgroundNotification
188  object:nil];
189  }
190  return self;
191 }
192 
193 - (void)setMaxRefreshRate:(double)refreshRate forceMax:(BOOL)forceMax {
194  // This is copied from vsync_waiter_ios.mm. The vsync waiter has display link scheduled on UI
195  // thread which does not trigger actual core animation frame. As a workaround FlutterMetalLayer
196  // has it's own displaylink scheduled on main thread, which is used to trigger core animation
197  // frame allowing for 120hz updates.
199  return;
200  }
201  double maxFrameRate = fmax(refreshRate, 60);
202  double minFrameRate = fmax(maxFrameRate / 2, 60);
203  if (@available(iOS 15.0, *)) {
204  _displayLink.preferredFrameRateRange =
205  CAFrameRateRangeMake(forceMax ? maxFrameRate : minFrameRate, maxFrameRate, maxFrameRate);
206  } else {
207  _displayLink.preferredFramesPerSecond = maxFrameRate;
208  }
209 }
210 
211 - (void)onDisplayLink:(CADisplayLink*)link {
212  _didSetContentsDuringThisDisplayLinkPeriod = NO;
213  // Do not pause immediately, this seems to prevent 120hz while touching.
214  if (_displayLinkPauseCountdown == 3) {
215  _displayLink.paused = YES;
216  if (_displayLinkForcedMaxRate) {
217  [self setMaxRefreshRate:DisplayLinkManager.displayRefreshRate forceMax:NO];
218  _displayLinkForcedMaxRate = NO;
219  }
220  } else {
221  ++_displayLinkPauseCountdown;
222  }
223 }
224 
225 - (BOOL)isKindOfClass:(Class)aClass {
226 #pragma clang diagnostic push
227 #pragma clang diagnostic ignored "-Wunguarded-availability-new"
228  // Pretend that we're a CAMetalLayer so that the rest of Flutter plays along
229  if ([aClass isEqual:[CAMetalLayer class]]) {
230  return YES;
231  }
232 #pragma clang diagnostic pop
233  return [super isKindOfClass:aClass];
234 }
235 
236 - (void)setDrawableSize:(CGSize)drawableSize {
237  [_availableTextures removeAllObjects];
238  _front = nil;
239  _totalTextures = 0;
240  _drawableSize = drawableSize;
241 }
242 
243 - (void)didEnterBackground:(id)notification {
244  [_availableTextures removeAllObjects];
245  _totalTextures = _front != nil ? 1 : 0;
246  _displayLink.paused = YES;
247 }
248 
249 - (CGSize)drawableSize {
250  return _drawableSize;
251 }
252 
253 - (IOSurface*)createIOSurface {
254  unsigned pixelFormat;
255  unsigned bytesPerElement;
256  if (self.pixelFormat == MTLPixelFormatRGBA16Float) {
257  pixelFormat = kCVPixelFormatType_64RGBAHalf;
258  bytesPerElement = 8;
259  } else if (self.pixelFormat == MTLPixelFormatBGRA8Unorm) {
260  pixelFormat = kCVPixelFormatType_32BGRA;
261  bytesPerElement = 4;
262  } else if (self.pixelFormat == MTLPixelFormatBGRA10_XR) {
263  pixelFormat = kCVPixelFormatType_40ARGBLEWideGamut;
264  bytesPerElement = 8;
265  } else {
266  FML_LOG(ERROR) << "Unsupported pixel format: " << self.pixelFormat;
267  return nil;
268  }
269  size_t bytesPerRow =
270  IOSurfaceAlignProperty(kIOSurfaceBytesPerRow, _drawableSize.width * bytesPerElement);
271  size_t totalBytes =
272  IOSurfaceAlignProperty(kIOSurfaceAllocSize, _drawableSize.height * bytesPerRow);
273  NSDictionary* options = @{
274  (id)kIOSurfaceWidth : @(_drawableSize.width),
275  (id)kIOSurfaceHeight : @(_drawableSize.height),
276  (id)kIOSurfacePixelFormat : @(pixelFormat),
277  (id)kIOSurfaceBytesPerElement : @(bytesPerElement),
278  (id)kIOSurfaceBytesPerRow : @(bytesPerRow),
279  (id)kIOSurfaceAllocSize : @(totalBytes),
280  };
281 
282  IOSurfaceRef res = IOSurfaceCreate((CFDictionaryRef)options);
283  if (res == nil) {
284  FML_LOG(ERROR) << "Failed to create IOSurface with options "
285  << options.debugDescription.UTF8String;
286  return nil;
287  }
288 
289  if (self.colorspace != nil) {
290  CFStringRef name = CGColorSpaceGetName(self.colorspace);
291  IOSurfaceSetValue(res, CFSTR("IOSurfaceColorSpace"), name);
292  } else {
293  IOSurfaceSetValue(res, CFSTR("IOSurfaceColorSpace"), kCGColorSpaceSRGB);
294  }
295  return (__bridge_transfer IOSurface*)res;
296 }
297 
298 - (FlutterTexture*)nextTexture {
299  CFTimeInterval start = CACurrentMediaTime();
300  while (true) {
301  FlutterTexture* texture = [self tryNextTexture];
302  if (texture != nil) {
303  return texture;
304  }
305  CFTimeInterval elapsed = CACurrentMediaTime() - start;
306  if (elapsed > 1.0) {
307  NSLog(@"Waited %f seconds for a drawable, giving up.", elapsed);
308  return nil;
309  }
310  }
311 }
312 
313 - (FlutterTexture*)tryNextTexture {
314  @synchronized(self) {
315  if (_front != nil && _front.waitingForCompletion) {
316  return nil;
317  }
318  if (_totalTextures < 3) {
319  ++_totalTextures;
320  IOSurface* surface = [self createIOSurface];
321  if (surface == nil) {
322  return nil;
323  }
324  MTLTextureDescriptor* textureDescriptor =
325  [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:_pixelFormat
326  width:_drawableSize.width
327  height:_drawableSize.height
328  mipmapped:NO];
329 
330  if (_framebufferOnly) {
331  textureDescriptor.usage = MTLTextureUsageRenderTarget;
332  } else {
333  textureDescriptor.usage =
334  MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite;
335  }
336  id<MTLTexture> texture = [self.device newTextureWithDescriptor:textureDescriptor
337  iosurface:(__bridge IOSurfaceRef)surface
338  plane:0];
339  FlutterTexture* flutterTexture = [[FlutterTexture alloc] initWithTexture:texture
340  surface:surface];
341  return flutterTexture;
342  } else {
343  // Prefer surface that is not in use and has been presented the longest
344  // time ago.
345  // When isInUse is false, the surface is definitely not used by the compositor.
346  // When isInUse is true, the surface may be used by the compositor.
347  // When both surfaces are in use, the one presented earlier will be returned.
348  // The assumption here is that the compositor is already aware of the
349  // newer texture and is unlikely to read from the older one, even though it
350  // has not decreased the use count yet (there seems to be certain latency).
351  FlutterTexture* res = nil;
352  for (FlutterTexture* texture in _availableTextures) {
353  if (res == nil) {
354  res = texture;
355  } else if (res.surface.isInUse && !texture.surface.isInUse) {
356  // prefer texture that is not in use.
357  res = texture;
358  } else if (res.surface.isInUse == texture.surface.isInUse &&
359  texture.presentedTime < res.presentedTime) {
360  // prefer texture with older presented time.
361  res = texture;
362  }
363  }
364  if (res != nil) {
365  [_availableTextures removeObject:res];
366  }
367  return res;
368  }
369  }
370 }
371 
372 - (id<CAMetalDrawable>)nextDrawable {
373  FlutterTexture* texture = [self nextTexture];
374  if (texture == nil) {
375  return nil;
376  }
377  FlutterDrawable* drawable = [[FlutterDrawable alloc] initWithTexture:texture
378  layer:self
379  drawableId:_nextDrawableId++];
380  return drawable;
381 }
382 
383 - (void)presentOnMainThread:(FlutterTexture*)texture {
384  // This is needed otherwise frame gets skipped on touch begin / end. Go figure.
385  // Might also be placebo
386  [self setNeedsDisplay];
387 
388  [CATransaction begin];
389  [CATransaction setDisableActions:YES];
390  self.contents = texture.surface;
391  [CATransaction commit];
392  _displayLink.paused = NO;
393  _displayLinkPauseCountdown = 0;
394  if (!_didSetContentsDuringThisDisplayLinkPeriod) {
395  _didSetContentsDuringThisDisplayLinkPeriod = YES;
396  } else if (!_displayLinkForcedMaxRate) {
397  _displayLinkForcedMaxRate = YES;
398  [self setMaxRefreshRate:DisplayLinkManager.displayRefreshRate forceMax:YES];
399  }
400 }
401 
402 - (void)presentTexture:(FlutterTexture*)texture {
403  @synchronized(self) {
404  if (_front != nil) {
405  [_availableTextures addObject:_front];
406  }
407  _front = texture;
408  texture.presentedTime = CACurrentMediaTime();
409  if ([NSThread isMainThread]) {
410  [self presentOnMainThread:texture];
411  } else {
412  // Core animation layers can only be updated on main thread.
413  dispatch_async(dispatch_get_main_queue(), ^{
414  [self presentOnMainThread:texture];
415  });
416  }
417  }
418 }
419 
420 - (void)returnTexture:(FlutterTexture*)texture {
421  @synchronized(self) {
422  [_availableTextures addObject:texture];
423  }
424 }
425 
426 + (BOOL)enabled {
427  static BOOL enabled = NO;
428  static BOOL didCheckInfoPlist = NO;
429  if (!didCheckInfoPlist) {
430  didCheckInfoPlist = YES;
431  NSNumber* use_flutter_metal_layer =
432  [[NSBundle mainBundle] objectForInfoDictionaryKey:@"FLTUseFlutterMetalLayer"];
433  if (use_flutter_metal_layer != nil && [use_flutter_metal_layer boolValue]) {
434  enabled = YES;
435  FML_LOG(WARNING) << "Using FlutterMetalLayer. This is an experimental feature.";
436  }
437  }
438  return enabled;
439 }
440 
441 @end
FlutterMetalLayer::wantsExtendedDynamicRangeContent
BOOL wantsExtendedDynamicRangeContent
Definition: FlutterMetalLayer.h:21
+[FlutterMetalLayer enabled]
BOOL enabled()
Definition: FlutterMetalLayer.mm:426
FlutterDrawable::_texture
FlutterTexture * _texture
Definition: FlutterMetalLayer.mm:89
FlutterTexture::_surface
IOSurface * _surface
Definition: FlutterMetalLayer.mm:60
FlutterDrawable
Definition: FlutterMetalLayer.mm:88
FlutterMetalLayer()::_preferredDevice
id< MTLDevice > _preferredDevice
Definition: FlutterMetalLayer.mm:27
FlutterDrawable::_drawableId
NSUInteger _drawableId
Definition: FlutterMetalLayer.mm:91
FlutterTexture::waitingForCompletion
BOOL waitingForCompletion
Definition: FlutterMetalLayer.mm:67
FlutterMetalLayer()::_didSetContentsDuringThisDisplayLinkPeriod
BOOL _didSetContentsDuringThisDisplayLinkPeriod
Definition: FlutterMetalLayer.mm:47
FlutterMacros.h
FlutterMetalLayer::framebufferOnly
BOOL framebufferOnly
Definition: FlutterMetalLayer.h:17
FlutterMetalLayer()::_totalTextures
NSUInteger _totalTextures
Definition: FlutterMetalLayer.mm:33
FlutterMetalLayer()::_front
FlutterTexture * _front
Definition: FlutterMetalLayer.mm:35
FlutterMetalLayer::colorspace
CGColorSpaceRef colorspace
Definition: FlutterMetalLayer.h:20
FlutterTexture::_presentedTime
CFTimeInterval _presentedTime
Definition: FlutterMetalLayer.mm:61
FlutterMetalDrawable-p
Definition: FlutterMetalLayer.h:33
FlutterMetalLayer()::_drawableSize
CGSize _drawableSize
Definition: FlutterMetalLayer.mm:28
_displayLink
CADisplayLink * _displayLink
Definition: vsync_waiter_ios.mm:64
FlutterMetalLayer()::_displayLinkPauseCountdown
NSUInteger _displayLinkPauseCountdown
Definition: FlutterMetalLayer.mm:40
FlutterMetalLayer::preferredDevice
id< MTLDevice > preferredDevice
Definition: FlutterMetalLayer.h:15
FlutterMetalLayer()::_availableTextures
NSMutableSet< FlutterTexture * > * _availableTextures
Definition: FlutterMetalLayer.mm:32
FlutterTexture
Definition: FlutterMetalLayer.mm:58
display_link_target
CFTimeInterval display_link_target
FlutterMetalLayer()::_nextDrawableId
NSUInteger _nextDrawableId
Definition: FlutterMetalLayer.mm:30
FlutterTexture::_texture
id< MTLTexture > _texture
Definition: FlutterMetalLayer.mm:59
FlutterTexture::presentedTime
CFTimeInterval presentedTime
Definition: FlutterMetalLayer.mm:66
FlutterMetalLayer()::_displayLink
CADisplayLink * _displayLink
Definition: FlutterMetalLayer.mm:39
FlutterTexture::surface
IOSurface * surface
Definition: FlutterMetalLayer.mm:65
FlutterMetalLayer::device
id< MTLDevice > device
Definition: FlutterMetalLayer.h:14
FlutterMetalLayer.h
FlutterDrawable::_presented
BOOL _presented
Definition: FlutterMetalLayer.mm:92
-[FlutterMetalLayer nextDrawable]
nullable id< CAMetalDrawable > nextDrawable()
Definition: FlutterMetalLayer.mm:372
FlutterMetalLayer::pixelFormat
MTLPixelFormat pixelFormat
Definition: FlutterMetalLayer.h:16
FlutterMetalLayer()::_displayLinkForcedMaxRate
BOOL _displayLinkForcedMaxRate
Definition: FlutterMetalLayer.mm:50
FLUTTER_ASSERT_ARC
Definition: FlutterChannelKeyResponder.mm:13
FlutterMetalLayer::drawableSize
CGSize drawableSize
Definition: FlutterMetalLayer.h:18
FlutterDrawable::_layer
__weak FlutterMetalLayer * _layer
Definition: FlutterMetalLayer.mm:90
FlutterTexture::texture
id< MTLTexture > texture
Definition: FlutterMetalLayer.mm:64
FlutterMetalLayer
Definition: FlutterMetalLayer.h:12