Flutter macOS Embedder
FlutterViewController.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
7 
8 #include <Carbon/Carbon.h>
9 #import <objc/message.h>
10 
11 #include "flutter/common/constants.h"
12 #include "flutter/shell/platform/embedder/embedder.h"
13 
23 
24 #pragma mark - Static types and data.
25 
26 namespace {
29 
30 // Use different device ID for mouse and pan/zoom events, since we can't differentiate the actual
31 // device (mouse v.s. trackpad).
32 static constexpr int32_t kMousePointerDeviceId = 0;
33 static constexpr int32_t kPointerPanZoomDeviceId = 1;
34 
35 // A trackpad touch following inertial scrolling should cause an inertia cancel
36 // event to be issued. Use a window of 50 milliseconds after the scroll to account
37 // for delays in event propagation observed in macOS Ventura.
38 static constexpr double kTrackpadTouchInertiaCancelWindowMs = 0.050;
39 
40 /**
41  * State tracking for mouse events, to adapt between the events coming from the system and the
42  * events that the embedding API expects.
43  */
44 struct MouseState {
45  /**
46  * The currently pressed buttons, as represented in FlutterPointerEvent.
47  */
48  int64_t buttons = 0;
49 
50  /**
51  * The accumulated gesture pan.
52  */
53  CGFloat delta_x = 0;
54  CGFloat delta_y = 0;
55 
56  /**
57  * The accumulated gesture zoom scale.
58  */
59  CGFloat scale = 0;
60 
61  /**
62  * The accumulated gesture rotation.
63  */
64  CGFloat rotation = 0;
65 
66  /**
67  * Whether or not a kAdd event has been sent (or sent again since the last kRemove if tracking is
68  * enabled). Used to determine whether to send a kAdd event before sending an incoming mouse
69  * event, since Flutter expects pointers to be added before events are sent for them.
70  */
71  bool flutter_state_is_added = false;
72 
73  /**
74  * Whether or not a kDown has been sent since the last kAdd/kUp.
75  */
76  bool flutter_state_is_down = false;
77 
78  /**
79  * Whether or not mouseExited: was received while a button was down. Cocoa's behavior when
80  * dragging out of a tracked area is to send an exit, then keep sending drag events until the last
81  * button is released. Flutter doesn't expect to receive events after a kRemove, so the kRemove
82  * for the exit needs to be delayed until after the last mouse button is released. If cursor
83  * returns back to the window while still dragging, the flag is cleared in mouseEntered:.
84  */
85  bool has_pending_exit = false;
86 
87  /*
88  * Whether or not a kPanZoomStart has been sent since the last kAdd/kPanZoomEnd.
89  */
90  bool flutter_state_is_pan_zoom_started = false;
91 
92  /**
93  * State of pan gesture.
94  */
95  NSEventPhase pan_gesture_phase = NSEventPhaseNone;
96 
97  /**
98  * State of scale gesture.
99  */
100  NSEventPhase scale_gesture_phase = NSEventPhaseNone;
101 
102  /**
103  * State of rotate gesture.
104  */
105  NSEventPhase rotate_gesture_phase = NSEventPhaseNone;
106 
107  /**
108  * Time of last scroll momentum event.
109  */
110  NSTimeInterval last_scroll_momentum_changed_time = 0;
111 
112  /**
113  * Resets all gesture state to default values.
114  */
115  void GestureReset() {
116  delta_x = 0;
117  delta_y = 0;
118  scale = 0;
119  rotation = 0;
120  flutter_state_is_pan_zoom_started = false;
121  pan_gesture_phase = NSEventPhaseNone;
122  scale_gesture_phase = NSEventPhaseNone;
123  rotate_gesture_phase = NSEventPhaseNone;
124  }
125 
126  /**
127  * Resets all state to default values.
128  */
129  void Reset() {
130  flutter_state_is_added = false;
131  flutter_state_is_down = false;
132  has_pending_exit = false;
133  buttons = 0;
134  }
135 };
136 
137 } // namespace
138 
139 #pragma mark - Private interface declaration.
140 
141 /**
142  * FlutterViewWrapper is a convenience class that wraps a FlutterView and provides
143  * a mechanism to attach AppKit views such as FlutterTextField without affecting
144  * the accessibility subtree of the wrapped FlutterView itself.
145  *
146  * The FlutterViewController uses this class to create its content view. When
147  * any of the accessibility services (e.g. VoiceOver) is turned on, the accessibility
148  * bridge creates FlutterTextFields that interact with the service. The bridge has to
149  * attach the FlutterTextField somewhere in the view hierarchy in order for the
150  * FlutterTextField to interact correctly with VoiceOver. Those FlutterTextFields
151  * will be attached to this view so that they won't affect the accessibility subtree
152  * of FlutterView.
153  */
154 @interface FlutterViewWrapper : NSView
155 
156 - (void)setBackgroundColor:(NSColor*)color;
157 
158 @end
159 
160 /**
161  * Private interface declaration for FlutterViewController.
162  */
164 
165 /**
166  * The tracking area used to generate hover events, if enabled.
167  */
168 @property(nonatomic) NSTrackingArea* trackingArea;
169 
170 /**
171  * The current state of the mouse and the sent mouse events.
172  */
173 @property(nonatomic) MouseState mouseState;
174 
175 /**
176  * Event monitor for keyUp events.
177  */
178 @property(nonatomic) id keyUpMonitor;
179 
180 /**
181  * Pointer to a keyboard manager, a hub that manages how key events are
182  * dispatched to various Flutter key responders, and whether the event is
183  * propagated to the next NSResponder.
184  */
185 @property(nonatomic, readonly, nonnull) FlutterKeyboardManager* keyboardManager;
186 
187 @property(nonatomic) KeyboardLayoutNotifier keyboardLayoutNotifier;
188 
189 @property(nonatomic) NSData* keyboardLayoutData;
190 
191 /**
192  * Starts running |engine|, including any initial setup.
193  */
194 - (BOOL)launchEngine;
195 
196 /**
197  * Updates |trackingArea| for the current tracking settings, creating it with
198  * the correct mode if tracking is enabled, or removing it if not.
199  */
200 - (void)configureTrackingArea;
201 
202 /**
203  * Creates and registers keyboard related components.
204  */
205 - (void)initializeKeyboard;
206 
207 /**
208  * Calls dispatchMouseEvent:phase: with a phase determined by self.mouseState.
209  *
210  * mouseState.buttons should be updated before calling this method.
211  */
212 - (void)dispatchMouseEvent:(nonnull NSEvent*)event;
213 
214 /**
215  * Calls dispatchMouseEvent:phase: with a phase determined by event.phase.
216  */
217 - (void)dispatchGestureEvent:(nonnull NSEvent*)event;
218 
219 /**
220  * Converts |event| to a FlutterPointerEvent with the given phase, and sends it to the engine.
221  */
222 - (void)dispatchMouseEvent:(nonnull NSEvent*)event phase:(FlutterPointerPhase)phase;
223 
224 /**
225  * Called when the active keyboard input source changes.
226  *
227  * Input sources may be simple keyboard layouts, or more complex input methods involving an IME,
228  * such as Chinese, Japanese, and Korean.
229  */
230 - (void)onKeyboardLayoutChanged;
231 
232 @end
233 
234 #pragma mark - FlutterViewWrapper implementation.
235 
236 /**
237  * NotificationCenter callback invoked on kTISNotifySelectedKeyboardInputSourceChanged events.
238  */
239 static void OnKeyboardLayoutChanged(CFNotificationCenterRef center,
240  void* observer,
241  CFStringRef name,
242  const void* object,
243  CFDictionaryRef userInfo) {
244  FlutterViewController* controller = (__bridge FlutterViewController*)observer;
245  if (controller != nil) {
246  [controller onKeyboardLayoutChanged];
247  }
248 }
249 
250 @implementation FlutterViewWrapper {
251  FlutterView* _flutterView;
253 }
254 
255 - (instancetype)initWithFlutterView:(FlutterView*)view
256  controller:(FlutterViewController*)controller {
257  self = [super initWithFrame:NSZeroRect];
258  if (self) {
259  _flutterView = view;
260  _controller = controller;
261  view.autoresizingMask = NSViewWidthSizable | NSViewHeightSizable;
262  [self addSubview:view];
263  }
264  return self;
265 }
266 
267 - (void)setBackgroundColor:(NSColor*)color {
268  [_flutterView setBackgroundColor:color];
269 }
270 
271 - (BOOL)performKeyEquivalent:(NSEvent*)event {
272  // Do not intercept the event if flutterView is not first responder, otherwise this would
273  // interfere with TextInputPlugin, which also handles key equivalents.
274  //
275  // Also do not intercept the event if key equivalent is a product of an event being
276  // redispatched by the TextInputPlugin, in which case it needs to bubble up so that menus
277  // can handle key equivalents.
278  if (self.window.firstResponder != _flutterView || [_controller isDispatchingKeyEvent:event]) {
279  return [super performKeyEquivalent:event];
280  }
281  [_flutterView keyDown:event];
282  return YES;
283 }
284 
285 - (NSArray*)accessibilityChildren {
286  return @[ _flutterView ];
287 }
288 
289 - (void)mouseDown:(NSEvent*)event {
290  // Work around an AppKit bug where mouseDown/mouseUp are not called on the view controller if the
291  // view is the content view of an NSPopover AND macOS's Reduced Transparency accessibility setting
292  // is enabled.
293  //
294  // This simply calls mouseDown on the next responder in the responder chain as the default
295  // implementation on NSResponder is documented to do.
296  //
297  // See: https://github.com/flutter/flutter/issues/115015
298  // See: http://www.openradar.me/FB12050037
299  // See: https://developer.apple.com/documentation/appkit/nsresponder/1524634-mousedown
300  [self.nextResponder mouseDown:event];
301 }
302 
303 - (void)mouseUp:(NSEvent*)event {
304  // Work around an AppKit bug where mouseDown/mouseUp are not called on the view controller if the
305  // view is the content view of an NSPopover AND macOS's Reduced Transparency accessibility setting
306  // is enabled.
307  //
308  // This simply calls mouseUp on the next responder in the responder chain as the default
309  // implementation on NSResponder is documented to do.
310  //
311  // See: https://github.com/flutter/flutter/issues/115015
312  // See: http://www.openradar.me/FB12050037
313  // See: https://developer.apple.com/documentation/appkit/nsresponder/1535349-mouseup
314  [self.nextResponder mouseUp:event];
315 }
316 
317 @end
318 
319 #pragma mark - FlutterViewController implementation.
320 
321 @implementation FlutterViewController {
322  // The project to run in this controller's engine.
324 
325  std::shared_ptr<flutter::AccessibilityBridgeMac> _bridge;
326 
327  // FlutterViewController does not actually uses the synchronizer, but only
328  // passes it to FlutterView.
330 }
331 
332 @synthesize viewIdentifier = _viewIdentifier;
333 @dynamic accessibilityBridge;
334 
335 /**
336  * Performs initialization that's common between the different init paths.
337  */
338 static void CommonInit(FlutterViewController* controller, FlutterEngine* engine) {
339  if (!engine) {
340  engine = [[FlutterEngine alloc] initWithName:@"io.flutter"
341  project:controller->_project
342  allowHeadlessExecution:NO];
343  }
344  NSCAssert(controller.engine == nil,
345  @"The FlutterViewController is unexpectedly attached to "
346  @"engine %@ before initialization.",
347  controller.engine);
348  [engine addViewController:controller];
349  NSCAssert(controller.engine != nil,
350  @"The FlutterViewController unexpectedly stays unattached after initialization. "
351  @"In unit tests, this is likely because either the FlutterViewController or "
352  @"the FlutterEngine is mocked. Please subclass these classes instead.",
353  controller.engine, controller.viewIdentifier);
354  controller->_mouseTrackingMode = kFlutterMouseTrackingModeInKeyWindow;
355  controller->_textInputPlugin = [[FlutterTextInputPlugin alloc] initWithViewController:controller];
356  [controller initializeKeyboard];
357  [controller notifySemanticsEnabledChanged];
358  // macOS fires this message when changing IMEs.
359  CFNotificationCenterRef cfCenter = CFNotificationCenterGetDistributedCenter();
360  __weak FlutterViewController* weakSelf = controller;
361  CFNotificationCenterAddObserver(cfCenter, (__bridge void*)weakSelf, OnKeyboardLayoutChanged,
362  kTISNotifySelectedKeyboardInputSourceChanged, NULL,
363  CFNotificationSuspensionBehaviorDeliverImmediately);
364 }
365 
366 - (instancetype)initWithCoder:(NSCoder*)coder {
367  self = [super initWithCoder:coder];
368  NSAssert(self, @"Super init cannot be nil");
369 
370  CommonInit(self, nil);
371  return self;
372 }
373 
374 - (instancetype)initWithNibName:(NSString*)nibNameOrNil bundle:(NSBundle*)nibBundleOrNil {
375  self = [super initWithNibName:nibNameOrNil bundle:nibBundleOrNil];
376  NSAssert(self, @"Super init cannot be nil");
377 
378  CommonInit(self, nil);
379  return self;
380 }
381 
382 - (instancetype)initWithProject:(nullable FlutterDartProject*)project {
383  self = [super initWithNibName:nil bundle:nil];
384  NSAssert(self, @"Super init cannot be nil");
385 
386  _project = project;
387  CommonInit(self, nil);
388  return self;
389 }
390 
391 - (instancetype)initWithEngine:(nonnull FlutterEngine*)engine
392  nibName:(nullable NSString*)nibName
393  bundle:(nullable NSBundle*)nibBundle {
394  NSAssert(engine != nil, @"Engine is required");
395 
396  self = [super initWithNibName:nibName bundle:nibBundle];
397  if (self) {
398  CommonInit(self, engine);
399  }
400 
401  return self;
402 }
403 
404 - (BOOL)isDispatchingKeyEvent:(NSEvent*)event {
405  return [_keyboardManager isDispatchingKeyEvent:event];
406 }
407 
408 - (void)loadView {
409  FlutterView* flutterView;
410  id<MTLDevice> device = _engine.renderer.device;
411  id<MTLCommandQueue> commandQueue = _engine.renderer.commandQueue;
412  if (!device || !commandQueue) {
413  NSLog(@"Unable to create FlutterView; no MTLDevice or MTLCommandQueue available.");
414  return;
415  }
416  flutterView = [self createFlutterViewWithMTLDevice:device commandQueue:commandQueue];
417  if (_backgroundColor != nil) {
418  [flutterView setBackgroundColor:_backgroundColor];
419  }
420  FlutterViewWrapper* wrapperView = [[FlutterViewWrapper alloc] initWithFlutterView:flutterView
421  controller:self];
422  self.view = wrapperView;
423  _flutterView = flutterView;
424 }
425 
426 - (void)viewDidLoad {
427  [self configureTrackingArea];
428  [self.view setAllowedTouchTypes:NSTouchTypeMaskIndirect];
429  [self.view setWantsRestingTouches:YES];
430  [_engine viewControllerViewDidLoad:self];
431 }
432 
433 - (void)viewWillAppear {
434  [super viewWillAppear];
435  if (!_engine.running) {
436  [self launchEngine];
437  }
438  [self listenForMetaModifiedKeyUpEvents];
439 }
440 
441 - (void)viewWillDisappear {
442  // Per Apple's documentation, it is discouraged to call removeMonitor: in dealloc, and it's
443  // recommended to be called earlier in the lifecycle.
444  [NSEvent removeMonitor:_keyUpMonitor];
445  _keyUpMonitor = nil;
446 }
447 
448 - (void)dealloc {
449  if ([self attached]) {
450  [_engine removeViewController:self];
451  }
452  CFNotificationCenterRef cfCenter = CFNotificationCenterGetDistributedCenter();
453  CFNotificationCenterRemoveEveryObserver(cfCenter, (__bridge void*)self);
454 }
455 
456 #pragma mark - Public methods
457 
458 - (void)setMouseTrackingMode:(FlutterMouseTrackingMode)mode {
459  if (_mouseTrackingMode == mode) {
460  return;
461  }
462  _mouseTrackingMode = mode;
463  [self configureTrackingArea];
464 }
465 
466 - (void)setBackgroundColor:(NSColor*)color {
467  _backgroundColor = color;
468  [_flutterView setBackgroundColor:_backgroundColor];
469 }
470 
471 - (FlutterViewIdentifier)viewIdentifier {
472  NSAssert([self attached], @"This view controller is not attached.");
473  return _viewIdentifier;
474 }
475 
476 - (void)onPreEngineRestart {
477  [self initializeKeyboard];
478 }
479 
480 - (void)notifySemanticsEnabledChanged {
481  BOOL mySemanticsEnabled = !!_bridge;
482  BOOL newSemanticsEnabled = _engine.semanticsEnabled;
483  if (newSemanticsEnabled == mySemanticsEnabled) {
484  return;
485  }
486  if (newSemanticsEnabled) {
487  _bridge = [self createAccessibilityBridgeWithEngine:_engine];
488  } else {
489  // Remove the accessibility children from flutter view before resetting the bridge.
490  _flutterView.accessibilityChildren = nil;
491  _bridge.reset();
492  }
493  NSAssert(newSemanticsEnabled == !!_bridge, @"Failed to update semantics for the view.");
494 }
495 
496 - (std::weak_ptr<flutter::AccessibilityBridgeMac>)accessibilityBridge {
497  return _bridge;
498 }
499 
500 - (void)setUpWithEngine:(FlutterEngine*)engine
501  viewIdentifier:(FlutterViewIdentifier)viewIdentifier
502  threadSynchronizer:(FlutterThreadSynchronizer*)threadSynchronizer {
503  NSAssert(_engine == nil, @"Already attached to an engine %@.", _engine);
504  _engine = engine;
505  _viewIdentifier = viewIdentifier;
506  _threadSynchronizer = threadSynchronizer;
507  [_threadSynchronizer registerView:_viewIdentifier];
508 }
509 
510 - (void)detachFromEngine {
511  NSAssert(_engine != nil, @"Not attached to any engine.");
512  [_threadSynchronizer deregisterView:_viewIdentifier];
513  _threadSynchronizer = nil;
514  _engine = nil;
515 }
516 
517 - (BOOL)attached {
518  return _engine != nil;
519 }
520 
521 - (void)updateSemantics:(const FlutterSemanticsUpdate2*)update {
522  NSAssert(_engine.semanticsEnabled, @"Semantics must be enabled.");
523  if (!_engine.semanticsEnabled) {
524  return;
525  }
526  for (size_t i = 0; i < update->node_count; i++) {
527  const FlutterSemanticsNode2* node = update->nodes[i];
528  _bridge->AddFlutterSemanticsNodeUpdate(*node);
529  }
530 
531  for (size_t i = 0; i < update->custom_action_count; i++) {
532  const FlutterSemanticsCustomAction2* action = update->custom_actions[i];
533  _bridge->AddFlutterSemanticsCustomActionUpdate(*action);
534  }
535 
536  _bridge->CommitUpdates();
537 
538  // Accessibility tree can only be used when the view is loaded.
539  if (!self.viewLoaded) {
540  return;
541  }
542  // Attaches the accessibility root to the flutter view.
543  auto root = _bridge->GetFlutterPlatformNodeDelegateFromID(0).lock();
544  if (root) {
545  if ([self.flutterView.accessibilityChildren count] == 0) {
546  NSAccessibilityElement* native_root = root->GetNativeViewAccessible();
547  self.flutterView.accessibilityChildren = @[ native_root ];
548  }
549  } else {
550  self.flutterView.accessibilityChildren = nil;
551  }
552 }
553 
554 #pragma mark - Private methods
555 
556 - (BOOL)launchEngine {
557  if (![_engine runWithEntrypoint:nil]) {
558  return NO;
559  }
560  return YES;
561 }
562 
563 // macOS does not call keyUp: on a key while the command key is pressed. This results in a loss
564 // of a key event once the modified key is released. This method registers the
565 // ViewController as a listener for a keyUp event before it's handled by NSApplication, and should
566 // NOT modify the event to avoid any unexpected behavior.
567 - (void)listenForMetaModifiedKeyUpEvents {
568  if (_keyUpMonitor != nil) {
569  // It is possible for [NSViewController viewWillAppear] to be invoked multiple times
570  // in a row. https://github.com/flutter/flutter/issues/105963
571  return;
572  }
573  FlutterViewController* __weak weakSelf = self;
574  _keyUpMonitor = [NSEvent
575  addLocalMonitorForEventsMatchingMask:NSEventMaskKeyUp
576  handler:^NSEvent*(NSEvent* event) {
577  // Intercept keyUp only for events triggered on the current
578  // view or textInputPlugin.
579  NSResponder* firstResponder = [[event window] firstResponder];
580  if (weakSelf.viewLoaded && weakSelf.flutterView &&
581  (firstResponder == weakSelf.flutterView ||
582  firstResponder == weakSelf.textInputPlugin) &&
583  ([event modifierFlags] & NSEventModifierFlagCommand) &&
584  ([event type] == NSEventTypeKeyUp)) {
585  [weakSelf keyUp:event];
586  }
587  return event;
588  }];
589 }
590 
591 - (void)configureTrackingArea {
592  if (!self.viewLoaded) {
593  // The viewDidLoad will call configureTrackingArea again when
594  // the view is actually loaded.
595  return;
596  }
597  if (_mouseTrackingMode != kFlutterMouseTrackingModeNone && self.flutterView) {
598  NSTrackingAreaOptions options = NSTrackingMouseEnteredAndExited | NSTrackingMouseMoved |
599  NSTrackingInVisibleRect | NSTrackingEnabledDuringMouseDrag;
600  switch (_mouseTrackingMode) {
601  case kFlutterMouseTrackingModeInKeyWindow:
602  options |= NSTrackingActiveInKeyWindow;
603  break;
604  case kFlutterMouseTrackingModeInActiveApp:
605  options |= NSTrackingActiveInActiveApp;
606  break;
607  case kFlutterMouseTrackingModeAlways:
608  options |= NSTrackingActiveAlways;
609  break;
610  default:
611  NSLog(@"Error: Unrecognized mouse tracking mode: %ld", _mouseTrackingMode);
612  return;
613  }
614  _trackingArea = [[NSTrackingArea alloc] initWithRect:NSZeroRect
615  options:options
616  owner:self
617  userInfo:nil];
618  [self.flutterView addTrackingArea:_trackingArea];
619  } else if (_trackingArea) {
620  [self.flutterView removeTrackingArea:_trackingArea];
621  _trackingArea = nil;
622  }
623 }
624 
625 - (void)initializeKeyboard {
626  // TODO(goderbauer): Seperate keyboard/textinput stuff into ViewController specific and Engine
627  // global parts. Move the global parts to FlutterEngine.
628  _keyboardManager = [[FlutterKeyboardManager alloc] initWithViewDelegate:self];
629 }
630 
631 - (void)dispatchMouseEvent:(nonnull NSEvent*)event {
632  FlutterPointerPhase phase = _mouseState.buttons == 0
633  ? (_mouseState.flutter_state_is_down ? kUp : kHover)
634  : (_mouseState.flutter_state_is_down ? kMove : kDown);
635  [self dispatchMouseEvent:event phase:phase];
636 }
637 
638 - (void)dispatchGestureEvent:(nonnull NSEvent*)event {
639  if (event.phase == NSEventPhaseBegan || event.phase == NSEventPhaseMayBegin) {
640  [self dispatchMouseEvent:event phase:kPanZoomStart];
641  } else if (event.phase == NSEventPhaseChanged) {
642  [self dispatchMouseEvent:event phase:kPanZoomUpdate];
643  } else if (event.phase == NSEventPhaseEnded || event.phase == NSEventPhaseCancelled) {
644  [self dispatchMouseEvent:event phase:kPanZoomEnd];
645  } else if (event.phase == NSEventPhaseNone && event.momentumPhase == NSEventPhaseNone) {
646  [self dispatchMouseEvent:event phase:kHover];
647  } else {
648  // Waiting until the first momentum change event is a workaround for an issue where
649  // touchesBegan: is called unexpectedly while in low power mode within the interval between
650  // momentum start and the first momentum change.
651  if (event.momentumPhase == NSEventPhaseChanged) {
652  _mouseState.last_scroll_momentum_changed_time = event.timestamp;
653  }
654  // Skip momentum update events, the framework will generate scroll momentum.
655  NSAssert(event.momentumPhase != NSEventPhaseNone,
656  @"Received gesture event with unexpected phase");
657  }
658 }
659 
660 - (void)dispatchMouseEvent:(NSEvent*)event phase:(FlutterPointerPhase)phase {
661  NSAssert(self.viewLoaded, @"View must be loaded before it handles the mouse event");
662  // There are edge cases where the system will deliver enter out of order relative to other
663  // events (e.g., drag out and back in, release, then click; mouseDown: will be called before
664  // mouseEntered:). Discard those events, since the add will already have been synthesized.
665  if (_mouseState.flutter_state_is_added && phase == kAdd) {
666  return;
667  }
668 
669  // Multiple gesture recognizers could be active at once, we can't send multiple kPanZoomStart.
670  // For example: rotation and magnification.
671  if (phase == kPanZoomStart || phase == kPanZoomEnd) {
672  if (event.type == NSEventTypeScrollWheel) {
673  _mouseState.pan_gesture_phase = event.phase;
674  } else if (event.type == NSEventTypeMagnify) {
675  _mouseState.scale_gesture_phase = event.phase;
676  } else if (event.type == NSEventTypeRotate) {
677  _mouseState.rotate_gesture_phase = event.phase;
678  }
679  }
680  if (phase == kPanZoomStart) {
681  if (event.type == NSEventTypeScrollWheel) {
682  // Ensure scroll inertia cancel event is not sent afterwards.
683  _mouseState.last_scroll_momentum_changed_time = 0;
684  }
685  if (_mouseState.flutter_state_is_pan_zoom_started) {
686  // Already started on a previous gesture type
687  return;
688  }
689  _mouseState.flutter_state_is_pan_zoom_started = true;
690  }
691  if (phase == kPanZoomEnd) {
692  if (!_mouseState.flutter_state_is_pan_zoom_started) {
693  // NSEventPhaseCancelled is sometimes received at incorrect times in the state
694  // machine, just ignore it here if it doesn't make sense
695  // (we have no active gesture to cancel).
696  NSAssert(event.phase == NSEventPhaseCancelled,
697  @"Received gesture event with unexpected phase");
698  return;
699  }
700  // NSEventPhase values are powers of two, we can use this to inspect merged phases.
701  NSEventPhase all_gestures_fields = _mouseState.pan_gesture_phase |
702  _mouseState.scale_gesture_phase |
703  _mouseState.rotate_gesture_phase;
704  NSEventPhase active_mask = NSEventPhaseBegan | NSEventPhaseChanged;
705  if ((all_gestures_fields & active_mask) != 0) {
706  // Even though this gesture type ended, a different type is still active.
707  return;
708  }
709  }
710 
711  // If a pointer added event hasn't been sent, synthesize one using this event for the basic
712  // information.
713  if (!_mouseState.flutter_state_is_added && phase != kAdd) {
714  // Only the values extracted for use in flutterEvent below matter, the rest are dummy values.
715  NSEvent* addEvent = [NSEvent enterExitEventWithType:NSEventTypeMouseEntered
716  location:event.locationInWindow
717  modifierFlags:0
718  timestamp:event.timestamp
719  windowNumber:event.windowNumber
720  context:nil
721  eventNumber:0
722  trackingNumber:0
723  userData:NULL];
724  [self dispatchMouseEvent:addEvent phase:kAdd];
725  }
726 
727  NSPoint locationInView = [self.flutterView convertPoint:event.locationInWindow fromView:nil];
728  NSPoint locationInBackingCoordinates = [self.flutterView convertPointToBacking:locationInView];
729  int32_t device = kMousePointerDeviceId;
730  FlutterPointerDeviceKind deviceKind = kFlutterPointerDeviceKindMouse;
731  if (phase == kPanZoomStart || phase == kPanZoomUpdate || phase == kPanZoomEnd) {
732  device = kPointerPanZoomDeviceId;
733  deviceKind = kFlutterPointerDeviceKindTrackpad;
734  }
735  FlutterPointerEvent flutterEvent = {
736  .struct_size = sizeof(flutterEvent),
737  .phase = phase,
738  .timestamp = static_cast<size_t>(event.timestamp * USEC_PER_SEC),
739  .x = locationInBackingCoordinates.x,
740  .y = -locationInBackingCoordinates.y, // convertPointToBacking makes this negative.
741  .device = device,
742  .device_kind = deviceKind,
743  // If a click triggered a synthesized kAdd, don't pass the buttons in that event.
744  .buttons = phase == kAdd ? 0 : _mouseState.buttons,
745  .view_id = static_cast<FlutterViewIdentifier>(_viewIdentifier),
746  };
747 
748  if (phase == kPanZoomUpdate) {
749  if (event.type == NSEventTypeScrollWheel) {
750  _mouseState.delta_x += event.scrollingDeltaX * self.flutterView.layer.contentsScale;
751  _mouseState.delta_y += event.scrollingDeltaY * self.flutterView.layer.contentsScale;
752  } else if (event.type == NSEventTypeMagnify) {
753  _mouseState.scale += event.magnification;
754  } else if (event.type == NSEventTypeRotate) {
755  _mouseState.rotation += event.rotation * (-M_PI / 180.0);
756  }
757  flutterEvent.pan_x = _mouseState.delta_x;
758  flutterEvent.pan_y = _mouseState.delta_y;
759  // Scale value needs to be normalized to range 0->infinity.
760  flutterEvent.scale = pow(2.0, _mouseState.scale);
761  flutterEvent.rotation = _mouseState.rotation;
762  } else if (phase == kPanZoomEnd) {
763  _mouseState.GestureReset();
764  } else if (phase != kPanZoomStart && event.type == NSEventTypeScrollWheel) {
765  flutterEvent.signal_kind = kFlutterPointerSignalKindScroll;
766 
767  double pixelsPerLine = 1.0;
768  if (!event.hasPreciseScrollingDeltas) {
769  // The scrollingDelta needs to be multiplied by the line height.
770  // CGEventSourceGetPixelsPerLine() will return 10, which will result in
771  // scrolling that is noticeably slower than in other applications.
772  // Using 40.0 as the multiplier to match Chromium.
773  // See https://source.chromium.org/chromium/chromium/src/+/main:ui/events/cocoa/events_mac.mm
774  pixelsPerLine = 40.0;
775  }
776  double scaleFactor = self.flutterView.layer.contentsScale;
777  // When mouse input is received while shift is pressed (regardless of
778  // any other pressed keys), Mac automatically flips the axis. Other
779  // platforms do not do this, so we flip it back to normalize the input
780  // received by the framework. The keyboard+mouse-scroll mechanism is exposed
781  // in the ScrollBehavior of the framework so developers can customize the
782  // behavior.
783  // At time of change, Apple does not expose any other type of API or signal
784  // that the X/Y axes have been flipped.
785  double scaledDeltaX = -event.scrollingDeltaX * pixelsPerLine * scaleFactor;
786  double scaledDeltaY = -event.scrollingDeltaY * pixelsPerLine * scaleFactor;
787  if (event.modifierFlags & NSShiftKeyMask) {
788  flutterEvent.scroll_delta_x = scaledDeltaY;
789  flutterEvent.scroll_delta_y = scaledDeltaX;
790  } else {
791  flutterEvent.scroll_delta_x = scaledDeltaX;
792  flutterEvent.scroll_delta_y = scaledDeltaY;
793  }
794  }
795 
796  [_keyboardManager syncModifiersIfNeeded:event.modifierFlags timestamp:event.timestamp];
797  [_engine sendPointerEvent:flutterEvent];
798 
799  // Update tracking of state as reported to Flutter.
800  if (phase == kDown) {
801  _mouseState.flutter_state_is_down = true;
802  } else if (phase == kUp) {
803  _mouseState.flutter_state_is_down = false;
804  if (_mouseState.has_pending_exit) {
805  [self dispatchMouseEvent:event phase:kRemove];
806  _mouseState.has_pending_exit = false;
807  }
808  } else if (phase == kAdd) {
809  _mouseState.flutter_state_is_added = true;
810  } else if (phase == kRemove) {
811  _mouseState.Reset();
812  }
813 }
814 
815 - (void)onAccessibilityStatusChanged:(BOOL)enabled {
816  if (!enabled && self.viewLoaded && [_textInputPlugin isFirstResponder]) {
817  // Normally TextInputPlugin, when editing, is child of FlutterViewWrapper.
818  // When accessiblity is enabled the TextInputPlugin gets added as an indirect
819  // child to FlutterTextField. When disabling the plugin needs to be reparented
820  // back.
821  [self.view addSubview:_textInputPlugin];
822  }
823 }
824 
825 - (std::shared_ptr<flutter::AccessibilityBridgeMac>)createAccessibilityBridgeWithEngine:
826  (nonnull FlutterEngine*)engine {
827  return std::make_shared<flutter::AccessibilityBridgeMac>(engine, self);
828 }
829 
830 - (nonnull FlutterView*)createFlutterViewWithMTLDevice:(id<MTLDevice>)device
831  commandQueue:(id<MTLCommandQueue>)commandQueue {
832  return [[FlutterView alloc] initWithMTLDevice:device
833  commandQueue:commandQueue
834  delegate:self
835  threadSynchronizer:_threadSynchronizer
836  viewIdentifier:_viewIdentifier];
837 }
838 
839 - (void)onKeyboardLayoutChanged {
840  _keyboardLayoutData = nil;
841  if (_keyboardLayoutNotifier != nil) {
843  }
844 }
845 
846 - (NSString*)lookupKeyForAsset:(NSString*)asset {
847  return [FlutterDartProject lookupKeyForAsset:asset];
848 }
849 
850 - (NSString*)lookupKeyForAsset:(NSString*)asset fromPackage:(NSString*)package {
851  return [FlutterDartProject lookupKeyForAsset:asset fromPackage:package];
852 }
853 
854 #pragma mark - FlutterViewDelegate
855 
856 /**
857  * Responds to view reshape by notifying the engine of the change in dimensions.
858  */
859 - (void)viewDidReshape:(NSView*)view {
860  FML_DCHECK(view == _flutterView);
861  [_engine updateWindowMetricsForViewController:self];
862 }
863 
864 - (BOOL)viewShouldAcceptFirstResponder:(NSView*)view {
865  FML_DCHECK(view == _flutterView);
866  // Only allow FlutterView to become first responder if TextInputPlugin is
867  // not active. Otherwise a mouse event inside FlutterView would cause the
868  // TextInputPlugin to lose first responder status.
869  return !_textInputPlugin.isFirstResponder;
870 }
871 
872 #pragma mark - FlutterPluginRegistry
873 
874 - (id<FlutterPluginRegistrar>)registrarForPlugin:(NSString*)pluginName {
875  return [_engine registrarForPlugin:pluginName];
876 }
877 
878 - (NSObject*)valuePublishedByPlugin:(NSString*)pluginKey {
879  return [_engine valuePublishedByPlugin:pluginKey];
880 }
881 
882 #pragma mark - FlutterKeyboardViewDelegate
883 
884 /**
885  * Returns the current Unicode layout data (kTISPropertyUnicodeKeyLayoutData).
886  *
887  * To use the returned data, convert it to CFDataRef first, finds its bytes
888  * with CFDataGetBytePtr, then reinterpret it into const UCKeyboardLayout*.
889  * It's returned in NSData* to enable auto reference count.
890  */
891 static NSData* CurrentKeyboardLayoutData() {
892  TISInputSourceRef source = TISCopyCurrentKeyboardInputSource();
893  CFTypeRef layout_data = TISGetInputSourceProperty(source, kTISPropertyUnicodeKeyLayoutData);
894  if (layout_data == nil) {
895  CFRelease(source);
896  // TISGetInputSourceProperty returns null with Japanese keyboard layout.
897  // Using TISCopyCurrentKeyboardLayoutInputSource to fix NULL return.
898  // https://github.com/microsoft/node-native-keymap/blob/5f0699ded00179410a14c0e1b0e089fe4df8e130/src/keyboard_mac.mm#L91
899  source = TISCopyCurrentKeyboardLayoutInputSource();
900  layout_data = TISGetInputSourceProperty(source, kTISPropertyUnicodeKeyLayoutData);
901  }
902  return (__bridge_transfer NSData*)CFRetain(layout_data);
903 }
904 
905 - (void)sendKeyEvent:(const FlutterKeyEvent&)event
906  callback:(nullable FlutterKeyEventCallback)callback
907  userData:(nullable void*)userData {
908  [_engine sendKeyEvent:event callback:callback userData:userData];
909 }
910 
911 - (id<FlutterBinaryMessenger>)getBinaryMessenger {
912  return _engine.binaryMessenger;
913 }
914 
915 - (BOOL)onTextInputKeyEvent:(nonnull NSEvent*)event {
916  return [_textInputPlugin handleKeyEvent:event];
917 }
918 
919 - (void)subscribeToKeyboardLayoutChange:(nullable KeyboardLayoutNotifier)callback {
920  _keyboardLayoutNotifier = callback;
921 }
922 
923 - (LayoutClue)lookUpLayoutForKeyCode:(uint16_t)keyCode shift:(BOOL)shift {
924  if (_keyboardLayoutData == nil) {
925  _keyboardLayoutData = CurrentKeyboardLayoutData();
926  }
927  const UCKeyboardLayout* layout = reinterpret_cast<const UCKeyboardLayout*>(
928  CFDataGetBytePtr((__bridge CFDataRef)_keyboardLayoutData));
929 
930  UInt32 deadKeyState = 0;
931  UniCharCount stringLength = 0;
932  UniChar resultChar;
933 
934  UInt32 modifierState = ((shift ? shiftKey : 0) >> 8) & 0xFF;
935  UInt32 keyboardType = LMGetKbdLast();
936 
937  bool isDeadKey = false;
938  OSStatus status =
939  UCKeyTranslate(layout, keyCode, kUCKeyActionDown, modifierState, keyboardType,
940  kUCKeyTranslateNoDeadKeysBit, &deadKeyState, 1, &stringLength, &resultChar);
941  // For dead keys, press the same key again to get the printable representation of the key.
942  if (status == noErr && stringLength == 0 && deadKeyState != 0) {
943  isDeadKey = true;
944  status =
945  UCKeyTranslate(layout, keyCode, kUCKeyActionDown, modifierState, keyboardType,
946  kUCKeyTranslateNoDeadKeysBit, &deadKeyState, 1, &stringLength, &resultChar);
947  }
948 
949  if (status == noErr && stringLength == 1 && !std::iscntrl(resultChar)) {
950  return LayoutClue{resultChar, isDeadKey};
951  }
952  return LayoutClue{0, false};
953 }
954 
955 - (nonnull NSDictionary*)getPressedState {
956  return [_keyboardManager getPressedState];
957 }
958 
959 #pragma mark - NSResponder
960 
961 - (BOOL)acceptsFirstResponder {
962  return YES;
963 }
964 
965 - (void)keyDown:(NSEvent*)event {
966  [_keyboardManager handleEvent:event];
967 }
968 
969 - (void)keyUp:(NSEvent*)event {
970  [_keyboardManager handleEvent:event];
971 }
972 
973 - (void)flagsChanged:(NSEvent*)event {
974  [_keyboardManager handleEvent:event];
975 }
976 
977 - (void)mouseEntered:(NSEvent*)event {
978  if (_mouseState.has_pending_exit) {
979  _mouseState.has_pending_exit = false;
980  } else {
981  [self dispatchMouseEvent:event phase:kAdd];
982  }
983 }
984 
985 - (void)mouseExited:(NSEvent*)event {
986  if (_mouseState.buttons != 0) {
987  _mouseState.has_pending_exit = true;
988  return;
989  }
990  [self dispatchMouseEvent:event phase:kRemove];
991 }
992 
993 - (void)mouseDown:(NSEvent*)event {
994  _mouseState.buttons |= kFlutterPointerButtonMousePrimary;
995  [self dispatchMouseEvent:event];
996 }
997 
998 - (void)mouseUp:(NSEvent*)event {
999  _mouseState.buttons &= ~static_cast<uint64_t>(kFlutterPointerButtonMousePrimary);
1000  [self dispatchMouseEvent:event];
1001 }
1002 
1003 - (void)mouseDragged:(NSEvent*)event {
1004  [self dispatchMouseEvent:event];
1005 }
1006 
1007 - (void)rightMouseDown:(NSEvent*)event {
1008  _mouseState.buttons |= kFlutterPointerButtonMouseSecondary;
1009  [self dispatchMouseEvent:event];
1010 }
1011 
1012 - (void)rightMouseUp:(NSEvent*)event {
1013  _mouseState.buttons &= ~static_cast<uint64_t>(kFlutterPointerButtonMouseSecondary);
1014  [self dispatchMouseEvent:event];
1015 }
1016 
1017 - (void)rightMouseDragged:(NSEvent*)event {
1018  [self dispatchMouseEvent:event];
1019 }
1020 
1021 - (void)otherMouseDown:(NSEvent*)event {
1022  _mouseState.buttons |= (1 << event.buttonNumber);
1023  [self dispatchMouseEvent:event];
1024 }
1025 
1026 - (void)otherMouseUp:(NSEvent*)event {
1027  _mouseState.buttons &= ~static_cast<uint64_t>(1 << event.buttonNumber);
1028  [self dispatchMouseEvent:event];
1029 }
1030 
1031 - (void)otherMouseDragged:(NSEvent*)event {
1032  [self dispatchMouseEvent:event];
1033 }
1034 
1035 - (void)mouseMoved:(NSEvent*)event {
1036  [self dispatchMouseEvent:event];
1037 }
1038 
1039 - (void)scrollWheel:(NSEvent*)event {
1040  [self dispatchGestureEvent:event];
1041 }
1042 
1043 - (void)magnifyWithEvent:(NSEvent*)event {
1044  [self dispatchGestureEvent:event];
1045 }
1046 
1047 - (void)rotateWithEvent:(NSEvent*)event {
1048  [self dispatchGestureEvent:event];
1049 }
1050 
1051 - (void)swipeWithEvent:(NSEvent*)event {
1052  // Not needed, it's handled by scrollWheel.
1053 }
1054 
1055 - (void)touchesBeganWithEvent:(NSEvent*)event {
1056  NSTouch* touch = event.allTouches.anyObject;
1057  if (touch != nil) {
1058  if ((event.timestamp - _mouseState.last_scroll_momentum_changed_time) <
1059  kTrackpadTouchInertiaCancelWindowMs) {
1060  // The trackpad has been touched following a scroll momentum event.
1061  // A scroll inertia cancel message should be sent to the framework.
1062  NSPoint locationInView = [self.flutterView convertPoint:event.locationInWindow fromView:nil];
1063  NSPoint locationInBackingCoordinates =
1064  [self.flutterView convertPointToBacking:locationInView];
1065  FlutterPointerEvent flutterEvent = {
1066  .struct_size = sizeof(flutterEvent),
1067  .timestamp = static_cast<size_t>(event.timestamp * USEC_PER_SEC),
1068  .x = locationInBackingCoordinates.x,
1069  .y = -locationInBackingCoordinates.y, // convertPointToBacking makes this negative.
1070  .device = kPointerPanZoomDeviceId,
1071  .signal_kind = kFlutterPointerSignalKindScrollInertiaCancel,
1072  .device_kind = kFlutterPointerDeviceKindTrackpad,
1073  .view_id = static_cast<FlutterViewIdentifier>(_viewIdentifier),
1074  };
1075 
1076  [_engine sendPointerEvent:flutterEvent];
1077  // Ensure no further scroll inertia cancel event will be sent.
1078  _mouseState.last_scroll_momentum_changed_time = 0;
1079  }
1080  }
1081 }
1082 
1083 @end
flutter::LayoutClue
Definition: FlutterKeyboardViewDelegate.h:20
FlutterEngine
Definition: FlutterEngine.h:30
FlutterViewController
Definition: FlutterViewController.h:73
FlutterEngine.h
FlutterViewWrapper
Definition: FlutterViewController.mm:154
FlutterEngine_Internal.h
+[FlutterDartProject lookupKeyForAsset:]
NSString * lookupKeyForAsset:(NSString *asset)
Definition: FlutterDartProject.mm:116
_keyboardLayoutNotifier
flutter::KeyboardLayoutNotifier _keyboardLayoutNotifier
Definition: FlutterKeyboardManagerTest.mm:243
_bridge
std::shared_ptr< flutter::AccessibilityBridgeMac > _bridge
Definition: FlutterViewController.mm:321
FlutterChannels.h
FlutterRenderer.h
_project
FlutterDartProject * _project
Definition: FlutterEngine.mm:409
FlutterViewController::engine
FlutterEngine * engine
Definition: FlutterViewController.h:78
FlutterPluginRegistrar-p
Definition: FlutterPluginRegistrarMacOS.h:28
FlutterKeyPrimaryResponder.h
-[FlutterView setBackgroundColor:]
void setBackgroundColor:(nonnull NSColor *color)
OnKeyboardLayoutChanged
static void OnKeyboardLayoutChanged(CFNotificationCenterRef center, void *observer, CFStringRef name, const void *object, CFDictionaryRef userInfo)
Definition: FlutterViewController.mm:239
_controller
__weak FlutterViewController * _controller
Definition: FlutterViewController.mm:250
FlutterThreadSynchronizer
Definition: FlutterThreadSynchronizer.h:18
flutter::KeyboardLayoutNotifier
void(^ KeyboardLayoutNotifier)()
Definition: FlutterKeyboardViewDelegate.h:16
FlutterTextInputPlugin
Definition: FlutterTextInputPlugin.h:27
FlutterCodecs.h
FlutterKeyboardManager.h
FlutterViewController_Internal.h
_threadSynchronizer
FlutterThreadSynchronizer * _threadSynchronizer
Definition: FlutterViewController.mm:329
FlutterViewController::viewIdentifier
FlutterViewIdentifier viewIdentifier
Definition: FlutterViewController.h:130
FlutterView
Definition: FlutterView.h:35
FlutterTextInputSemanticsObject.h
+[FlutterDartProject lookupKeyForAsset:fromPackage:]
NSString * lookupKeyForAsset:fromPackage:(NSString *asset,[fromPackage] NSString *package)
Definition: FlutterDartProject.mm:125
FlutterDartProject
Definition: FlutterDartProject.mm:24
FlutterKeyboardManager
Definition: FlutterKeyboardManager.h:27
FlutterBinaryMessenger-p
Definition: FlutterBinaryMessenger.h:49
FlutterView.h
FlutterViewIdentifier
int64_t FlutterViewIdentifier
Definition: FlutterViewController.h:21
FlutterViewDelegate-p
Definition: FlutterView.h:18
FlutterViewController.h