Flutter macOS Embedder
FlutterViewController.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
7 
8 #include <Carbon/Carbon.h>
9 #import <objc/message.h>
10 
20 #import "flutter/shell/platform/embedder/embedder.h"
21 
22 #pragma mark - Static types and data.
23 
24 namespace {
27 
28 // Use different device ID for mouse and pan/zoom events, since we can't differentiate the actual
29 // device (mouse v.s. trackpad).
30 static constexpr int32_t kMousePointerDeviceId = 0;
31 static constexpr int32_t kPointerPanZoomDeviceId = 1;
32 
33 // A trackpad touch following inertial scrolling should cause an inertia cancel
34 // event to be issued. Use a window of 50 milliseconds after the scroll to account
35 // for delays in event propagation observed in macOS Ventura.
36 static constexpr double kTrackpadTouchInertiaCancelWindowMs = 0.050;
37 
38 /**
39  * State tracking for mouse events, to adapt between the events coming from the system and the
40  * events that the embedding API expects.
41  */
42 struct MouseState {
43  /**
44  * The currently pressed buttons, as represented in FlutterPointerEvent.
45  */
46  int64_t buttons = 0;
47 
48  /**
49  * The accumulated gesture pan.
50  */
51  CGFloat delta_x = 0;
52  CGFloat delta_y = 0;
53 
54  /**
55  * The accumulated gesture zoom scale.
56  */
57  CGFloat scale = 0;
58 
59  /**
60  * The accumulated gesture rotation.
61  */
62  CGFloat rotation = 0;
63 
64  /**
65  * Whether or not a kAdd event has been sent (or sent again since the last kRemove if tracking is
66  * enabled). Used to determine whether to send a kAdd event before sending an incoming mouse
67  * event, since Flutter expects pointers to be added before events are sent for them.
68  */
69  bool flutter_state_is_added = false;
70 
71  /**
72  * Whether or not a kDown has been sent since the last kAdd/kUp.
73  */
74  bool flutter_state_is_down = false;
75 
76  /**
77  * Whether or not mouseExited: was received while a button was down. Cocoa's behavior when
78  * dragging out of a tracked area is to send an exit, then keep sending drag events until the last
79  * button is released. Flutter doesn't expect to receive events after a kRemove, so the kRemove
80  * for the exit needs to be delayed until after the last mouse button is released. If cursor
81  * returns back to the window while still dragging, the flag is cleared in mouseEntered:.
82  */
83  bool has_pending_exit = false;
84 
85  /*
86  * Whether or not a kPanZoomStart has been sent since the last kAdd/kPanZoomEnd.
87  */
88  bool flutter_state_is_pan_zoom_started = false;
89 
90  /**
91  * State of pan gesture.
92  */
93  NSEventPhase pan_gesture_phase = NSEventPhaseNone;
94 
95  /**
96  * State of scale gesture.
97  */
98  NSEventPhase scale_gesture_phase = NSEventPhaseNone;
99 
100  /**
101  * State of rotate gesture.
102  */
103  NSEventPhase rotate_gesture_phase = NSEventPhaseNone;
104 
105  /**
106  * Time of last scroll momentum event.
107  */
108  NSTimeInterval last_scroll_momentum_changed_time = 0;
109 
110  /**
111  * Resets all gesture state to default values.
112  */
113  void GestureReset() {
114  delta_x = 0;
115  delta_y = 0;
116  scale = 0;
117  rotation = 0;
118  flutter_state_is_pan_zoom_started = false;
119  pan_gesture_phase = NSEventPhaseNone;
120  scale_gesture_phase = NSEventPhaseNone;
121  rotate_gesture_phase = NSEventPhaseNone;
122  }
123 
124  /**
125  * Resets all state to default values.
126  */
127  void Reset() {
128  flutter_state_is_added = false;
129  flutter_state_is_down = false;
130  has_pending_exit = false;
131  buttons = 0;
132  GestureReset();
133  }
134 };
135 
136 } // namespace
137 
138 #pragma mark - Private interface declaration.
139 
140 /**
141  * FlutterViewWrapper is a convenience class that wraps a FlutterView and provides
142  * a mechanism to attach AppKit views such as FlutterTextField without affecting
143  * the accessibility subtree of the wrapped FlutterView itself.
144  *
145  * The FlutterViewController uses this class to create its content view. When
146  * any of the accessibility services (e.g. VoiceOver) is turned on, the accessibility
147  * bridge creates FlutterTextFields that interact with the service. The bridge has to
148  * attach the FlutterTextField somewhere in the view hierarchy in order for the
149  * FlutterTextField to interact correctly with VoiceOver. Those FlutterTextFields
150  * will be attached to this view so that they won't affect the accessibility subtree
151  * of FlutterView.
152  */
153 @interface FlutterViewWrapper : NSView
154 
155 - (void)setBackgroundColor:(NSColor*)color;
156 
157 @end
158 
159 /**
160  * Private interface declaration for FlutterViewController.
161  */
163 
164 /**
165  * The tracking area used to generate hover events, if enabled.
166  */
167 @property(nonatomic) NSTrackingArea* trackingArea;
168 
169 /**
170  * The current state of the mouse and the sent mouse events.
171  */
172 @property(nonatomic) MouseState mouseState;
173 
174 /**
175  * Event monitor for keyUp events.
176  */
177 @property(nonatomic) id keyUpMonitor;
178 
179 /**
180  * Pointer to a keyboard manager, a hub that manages how key events are
181  * dispatched to various Flutter key responders, and whether the event is
182  * propagated to the next NSResponder.
183  */
184 @property(nonatomic, readonly, nonnull) FlutterKeyboardManager* keyboardManager;
185 
186 @property(nonatomic) KeyboardLayoutNotifier keyboardLayoutNotifier;
187 
188 @property(nonatomic) NSData* keyboardLayoutData;
189 
190 /**
191  * Starts running |engine|, including any initial setup.
192  */
193 - (BOOL)launchEngine;
194 
195 /**
196  * Updates |trackingArea| for the current tracking settings, creating it with
197  * the correct mode if tracking is enabled, or removing it if not.
198  */
199 - (void)configureTrackingArea;
200 
201 /**
202  * Creates and registers keyboard related components.
203  */
204 - (void)initializeKeyboard;
205 
206 /**
207  * Calls dispatchMouseEvent:phase: with a phase determined by self.mouseState.
208  *
209  * mouseState.buttons should be updated before calling this method.
210  */
211 - (void)dispatchMouseEvent:(nonnull NSEvent*)event;
212 
213 /**
214  * Calls dispatchMouseEvent:phase: with a phase determined by event.phase.
215  */
216 - (void)dispatchGestureEvent:(nonnull NSEvent*)event;
217 
218 /**
219  * Converts |event| to a FlutterPointerEvent with the given phase, and sends it to the engine.
220  */
221 - (void)dispatchMouseEvent:(nonnull NSEvent*)event phase:(FlutterPointerPhase)phase;
222 
223 /**
224  * Called when the active keyboard input source changes.
225  *
226  * Input sources may be simple keyboard layouts, or more complex input methods involving an IME,
227  * such as Chinese, Japanese, and Korean.
228  */
229 - (void)onKeyboardLayoutChanged;
230 
231 @end
232 
233 #pragma mark - FlutterViewWrapper implementation.
234 
235 /**
236  * NotificationCenter callback invoked on kTISNotifySelectedKeyboardInputSourceChanged events.
237  */
238 static void OnKeyboardLayoutChanged(CFNotificationCenterRef center,
239  void* observer,
240  CFStringRef name,
241  const void* object,
242  CFDictionaryRef userInfo) {
243  FlutterViewController* controller = (__bridge FlutterViewController*)observer;
244  if (controller != nil) {
245  [controller onKeyboardLayoutChanged];
246  }
247 }
248 
249 @implementation FlutterViewWrapper {
250  FlutterView* _flutterView;
252 }
253 
254 - (instancetype)initWithFlutterView:(FlutterView*)view
255  controller:(FlutterViewController*)controller {
256  self = [super initWithFrame:NSZeroRect];
257  if (self) {
258  _flutterView = view;
259  _controller = controller;
260  view.autoresizingMask = NSViewWidthSizable | NSViewHeightSizable;
261  [self addSubview:view];
262  }
263  return self;
264 }
265 
266 - (void)setBackgroundColor:(NSColor*)color {
267  [_flutterView setBackgroundColor:color];
268 }
269 
270 - (BOOL)performKeyEquivalent:(NSEvent*)event {
271  // Do not intercept the event if flutterView is not first responder, otherwise this would
272  // interfere with TextInputPlugin, which also handles key equivalents.
273  //
274  // Also do not intercept the event if key equivalent is a product of an event being
275  // redispatched by the TextInputPlugin, in which case it needs to bubble up so that menus
276  // can handle key equivalents.
277  if (self.window.firstResponder != _flutterView || [_controller isDispatchingKeyEvent:event]) {
278  return [super performKeyEquivalent:event];
279  }
280  [_flutterView keyDown:event];
281  return YES;
282 }
283 
284 - (NSArray*)accessibilityChildren {
285  return @[ _flutterView ];
286 }
287 
288 - (void)mouseDown:(NSEvent*)event {
289  // Work around an AppKit bug where mouseDown/mouseUp are not called on the view controller if the
290  // view is the content view of an NSPopover AND macOS's Reduced Transparency accessibility setting
291  // is enabled.
292  //
293  // This simply calls mouseDown on the next responder in the responder chain as the default
294  // implementation on NSResponder is documented to do.
295  //
296  // See: https://github.com/flutter/flutter/issues/115015
297  // See: http://www.openradar.me/FB12050037
298  // See: https://developer.apple.com/documentation/appkit/nsresponder/1524634-mousedown
299  [self.nextResponder mouseDown:event];
300 }
301 
302 - (void)mouseUp:(NSEvent*)event {
303  // Work around an AppKit bug where mouseDown/mouseUp are not called on the view controller if the
304  // view is the content view of an NSPopover AND macOS's Reduced Transparency accessibility setting
305  // is enabled.
306  //
307  // This simply calls mouseUp on the next responder in the responder chain as the default
308  // implementation on NSResponder is documented to do.
309  //
310  // See: https://github.com/flutter/flutter/issues/115015
311  // See: http://www.openradar.me/FB12050037
312  // See: https://developer.apple.com/documentation/appkit/nsresponder/1535349-mouseup
313  [self.nextResponder mouseUp:event];
314 }
315 
316 @end
317 
318 #pragma mark - FlutterViewController implementation.
319 
320 @implementation FlutterViewController {
321  // The project to run in this controller's engine.
323 
324  std::shared_ptr<flutter::AccessibilityBridgeMac> _bridge;
325 
327 
328  // FlutterViewController does not actually uses the synchronizer, but only
329  // passes it to FlutterView.
331 }
332 
333 @synthesize viewId = _viewId;
334 @dynamic accessibilityBridge;
335 
336 /**
337  * Performs initialization that's common between the different init paths.
338  */
339 static void CommonInit(FlutterViewController* controller, FlutterEngine* engine) {
340  if (!engine) {
341  engine = [[FlutterEngine alloc] initWithName:@"io.flutter"
342  project:controller->_project
343  allowHeadlessExecution:NO];
344  }
345  NSCAssert(controller.engine == nil,
346  @"The FlutterViewController is unexpectedly attached to "
347  @"engine %@ before initialization.",
348  controller.engine);
349  [engine addViewController:controller];
350  NSCAssert(controller.engine != nil,
351  @"The FlutterViewController unexpectedly stays unattached after initialization. "
352  @"In unit tests, this is likely because either the FlutterViewController or "
353  @"the FlutterEngine is mocked. Please subclass these classes instead.",
354  controller.engine, controller.viewId);
355  controller->_mouseTrackingMode = kFlutterMouseTrackingModeInKeyWindow;
356  controller->_textInputPlugin = [[FlutterTextInputPlugin alloc] initWithViewController:controller];
357  [controller initializeKeyboard];
358  [controller notifySemanticsEnabledChanged];
359  // macOS fires this message when changing IMEs.
360  CFNotificationCenterRef cfCenter = CFNotificationCenterGetDistributedCenter();
361  __weak FlutterViewController* weakSelf = controller;
362  CFNotificationCenterAddObserver(cfCenter, (__bridge void*)weakSelf, OnKeyboardLayoutChanged,
363  kTISNotifySelectedKeyboardInputSourceChanged, NULL,
364  CFNotificationSuspensionBehaviorDeliverImmediately);
365 }
366 
367 - (instancetype)initWithCoder:(NSCoder*)coder {
368  self = [super initWithCoder:coder];
369  NSAssert(self, @"Super init cannot be nil");
370 
371  CommonInit(self, nil);
372  return self;
373 }
374 
375 - (instancetype)initWithNibName:(NSString*)nibNameOrNil bundle:(NSBundle*)nibBundleOrNil {
376  self = [super initWithNibName:nibNameOrNil bundle:nibBundleOrNil];
377  NSAssert(self, @"Super init cannot be nil");
378 
379  CommonInit(self, nil);
380  return self;
381 }
382 
383 - (instancetype)initWithProject:(nullable FlutterDartProject*)project {
384  self = [super initWithNibName:nil bundle:nil];
385  NSAssert(self, @"Super init cannot be nil");
386 
387  _project = project;
388  CommonInit(self, nil);
389  return self;
390 }
391 
392 - (instancetype)initWithEngine:(nonnull FlutterEngine*)engine
393  nibName:(nullable NSString*)nibName
394  bundle:(nullable NSBundle*)nibBundle {
395  NSAssert(engine != nil, @"Engine is required");
396 
397  self = [super initWithNibName:nibName bundle:nibBundle];
398  if (self) {
399  CommonInit(self, engine);
400  }
401 
402  return self;
403 }
404 
405 - (BOOL)isDispatchingKeyEvent:(NSEvent*)event {
406  return [_keyboardManager isDispatchingKeyEvent:event];
407 }
408 
409 - (void)loadView {
410  FlutterView* flutterView;
411  id<MTLDevice> device = _engine.renderer.device;
412  id<MTLCommandQueue> commandQueue = _engine.renderer.commandQueue;
413  if (!device || !commandQueue) {
414  NSLog(@"Unable to create FlutterView; no MTLDevice or MTLCommandQueue available.");
415  return;
416  }
417  flutterView = [self createFlutterViewWithMTLDevice:device commandQueue:commandQueue];
418  if (_backgroundColor != nil) {
419  [flutterView setBackgroundColor:_backgroundColor];
420  }
421  FlutterViewWrapper* wrapperView = [[FlutterViewWrapper alloc] initWithFlutterView:flutterView
422  controller:self];
423  self.view = wrapperView;
424  _flutterView = flutterView;
425 }
426 
427 - (void)viewDidLoad {
428  [self configureTrackingArea];
429  [self.view setAllowedTouchTypes:NSTouchTypeMaskIndirect];
430  [self.view setWantsRestingTouches:YES];
431  [_engine viewControllerViewDidLoad:self];
432 }
433 
434 - (void)viewWillAppear {
435  [super viewWillAppear];
436  if (!_engine.running) {
437  [self launchEngine];
438  }
439  [self listenForMetaModifiedKeyUpEvents];
440 }
441 
442 - (void)viewWillDisappear {
443  // Per Apple's documentation, it is discouraged to call removeMonitor: in dealloc, and it's
444  // recommended to be called earlier in the lifecycle.
445  [NSEvent removeMonitor:_keyUpMonitor];
446  _keyUpMonitor = nil;
447 }
448 
449 - (void)dealloc {
450  if ([self attached]) {
451  [_engine removeViewController:self];
452  }
453  CFNotificationCenterRef cfCenter = CFNotificationCenterGetDistributedCenter();
454  CFNotificationCenterRemoveEveryObserver(cfCenter, (__bridge void*)self);
455 }
456 
457 #pragma mark - Public methods
458 
459 - (void)setMouseTrackingMode:(FlutterMouseTrackingMode)mode {
460  if (_mouseTrackingMode == mode) {
461  return;
462  }
463  _mouseTrackingMode = mode;
464  [self configureTrackingArea];
465 }
466 
467 - (void)setBackgroundColor:(NSColor*)color {
468  _backgroundColor = color;
469  [_flutterView setBackgroundColor:_backgroundColor];
470 }
471 
472 - (FlutterViewId)viewId {
473  NSAssert([self attached], @"This view controller is not attached.");
474  return _viewId;
475 }
476 
477 - (void)onPreEngineRestart {
478  [self initializeKeyboard];
479 }
480 
481 - (void)notifySemanticsEnabledChanged {
482  BOOL mySemanticsEnabled = !!_bridge;
483  BOOL newSemanticsEnabled = _engine.semanticsEnabled;
484  if (newSemanticsEnabled == mySemanticsEnabled) {
485  return;
486  }
487  if (newSemanticsEnabled) {
488  _bridge = [self createAccessibilityBridgeWithEngine:_engine];
489  } else {
490  // Remove the accessibility children from flutter view before resetting the bridge.
491  _flutterView.accessibilityChildren = nil;
492  _bridge.reset();
493  }
494  NSAssert(newSemanticsEnabled == !!_bridge, @"Failed to update semantics for the view.");
495 }
496 
497 - (std::weak_ptr<flutter::AccessibilityBridgeMac>)accessibilityBridge {
498  return _bridge;
499 }
500 
501 - (void)setUpWithEngine:(FlutterEngine*)engine
502  viewId:(FlutterViewId)viewId
503  threadSynchronizer:(FlutterThreadSynchronizer*)threadSynchronizer {
504  NSAssert(_engine == nil, @"Already attached to an engine %@.", _engine);
505  _engine = engine;
506  _viewId = viewId;
507  _threadSynchronizer = threadSynchronizer;
508  [_threadSynchronizer registerView:_viewId];
509 }
510 
511 - (void)detachFromEngine {
512  NSAssert(_engine != nil, @"Not attached to any engine.");
513  [_threadSynchronizer deregisterView:_viewId];
514  _threadSynchronizer = nil;
515  _engine = nil;
516 }
517 
518 - (BOOL)attached {
519  return _engine != nil;
520 }
521 
522 - (void)updateSemantics:(const FlutterSemanticsUpdate2*)update {
523  NSAssert(_engine.semanticsEnabled, @"Semantics must be enabled.");
524  if (!_engine.semanticsEnabled) {
525  return;
526  }
527  for (size_t i = 0; i < update->node_count; i++) {
528  const FlutterSemanticsNode2* node = update->nodes[i];
529  _bridge->AddFlutterSemanticsNodeUpdate(*node);
530  }
531 
532  for (size_t i = 0; i < update->custom_action_count; i++) {
533  const FlutterSemanticsCustomAction2* action = update->custom_actions[i];
534  _bridge->AddFlutterSemanticsCustomActionUpdate(*action);
535  }
536 
537  _bridge->CommitUpdates();
538 
539  // Accessibility tree can only be used when the view is loaded.
540  if (!self.viewLoaded) {
541  return;
542  }
543  // Attaches the accessibility root to the flutter view.
544  auto root = _bridge->GetFlutterPlatformNodeDelegateFromID(0).lock();
545  if (root) {
546  if ([self.flutterView.accessibilityChildren count] == 0) {
547  NSAccessibilityElement* native_root = root->GetNativeViewAccessible();
548  self.flutterView.accessibilityChildren = @[ native_root ];
549  }
550  } else {
551  self.flutterView.accessibilityChildren = nil;
552  }
553 }
554 
555 #pragma mark - Private methods
556 
557 - (BOOL)launchEngine {
558  if (![_engine runWithEntrypoint:nil]) {
559  return NO;
560  }
561  return YES;
562 }
563 
564 // macOS does not call keyUp: on a key while the command key is pressed. This results in a loss
565 // of a key event once the modified key is released. This method registers the
566 // ViewController as a listener for a keyUp event before it's handled by NSApplication, and should
567 // NOT modify the event to avoid any unexpected behavior.
568 - (void)listenForMetaModifiedKeyUpEvents {
569  if (_keyUpMonitor != nil) {
570  // It is possible for [NSViewController viewWillAppear] to be invoked multiple times
571  // in a row. https://github.com/flutter/flutter/issues/105963
572  return;
573  }
574  FlutterViewController* __weak weakSelf = self;
575  _keyUpMonitor = [NSEvent
576  addLocalMonitorForEventsMatchingMask:NSEventMaskKeyUp
577  handler:^NSEvent*(NSEvent* event) {
578  // Intercept keyUp only for events triggered on the current
579  // view or textInputPlugin.
580  NSResponder* firstResponder = [[event window] firstResponder];
581  if (weakSelf.viewLoaded && weakSelf.flutterView &&
582  (firstResponder == weakSelf.flutterView ||
583  firstResponder == weakSelf.textInputPlugin) &&
584  ([event modifierFlags] & NSEventModifierFlagCommand) &&
585  ([event type] == NSEventTypeKeyUp)) {
586  [weakSelf keyUp:event];
587  }
588  return event;
589  }];
590 }
591 
592 - (void)configureTrackingArea {
593  if (!self.viewLoaded) {
594  // The viewDidLoad will call configureTrackingArea again when
595  // the view is actually loaded.
596  return;
597  }
598  if (_mouseTrackingMode != kFlutterMouseTrackingModeNone && self.flutterView) {
599  NSTrackingAreaOptions options = NSTrackingMouseEnteredAndExited | NSTrackingMouseMoved |
600  NSTrackingInVisibleRect | NSTrackingEnabledDuringMouseDrag;
601  switch (_mouseTrackingMode) {
602  case kFlutterMouseTrackingModeInKeyWindow:
603  options |= NSTrackingActiveInKeyWindow;
604  break;
605  case kFlutterMouseTrackingModeInActiveApp:
606  options |= NSTrackingActiveInActiveApp;
607  break;
608  case kFlutterMouseTrackingModeAlways:
609  options |= NSTrackingActiveAlways;
610  break;
611  default:
612  NSLog(@"Error: Unrecognized mouse tracking mode: %ld", _mouseTrackingMode);
613  return;
614  }
615  _trackingArea = [[NSTrackingArea alloc] initWithRect:NSZeroRect
616  options:options
617  owner:self
618  userInfo:nil];
619  [self.flutterView addTrackingArea:_trackingArea];
620  } else if (_trackingArea) {
621  [self.flutterView removeTrackingArea:_trackingArea];
622  _trackingArea = nil;
623  }
624 }
625 
626 - (void)initializeKeyboard {
627  // TODO(goderbauer): Seperate keyboard/textinput stuff into ViewController specific and Engine
628  // global parts. Move the global parts to FlutterEngine.
629  _keyboardManager = [[FlutterKeyboardManager alloc] initWithViewDelegate:self];
630 }
631 
632 - (void)dispatchMouseEvent:(nonnull NSEvent*)event {
633  FlutterPointerPhase phase = _mouseState.buttons == 0
634  ? (_mouseState.flutter_state_is_down ? kUp : kHover)
635  : (_mouseState.flutter_state_is_down ? kMove : kDown);
636  [self dispatchMouseEvent:event phase:phase];
637 }
638 
639 - (void)dispatchGestureEvent:(nonnull NSEvent*)event {
640  if (event.phase == NSEventPhaseBegan || event.phase == NSEventPhaseMayBegin) {
641  [self dispatchMouseEvent:event phase:kPanZoomStart];
642  } else if (event.phase == NSEventPhaseChanged) {
643  [self dispatchMouseEvent:event phase:kPanZoomUpdate];
644  } else if (event.phase == NSEventPhaseEnded || event.phase == NSEventPhaseCancelled) {
645  [self dispatchMouseEvent:event phase:kPanZoomEnd];
646  } else if (event.phase == NSEventPhaseNone && event.momentumPhase == NSEventPhaseNone) {
647  [self dispatchMouseEvent:event phase:kHover];
648  } else {
649  // Waiting until the first momentum change event is a workaround for an issue where
650  // touchesBegan: is called unexpectedly while in low power mode within the interval between
651  // momentum start and the first momentum change.
652  if (event.momentumPhase == NSEventPhaseChanged) {
653  _mouseState.last_scroll_momentum_changed_time = event.timestamp;
654  }
655  // Skip momentum update events, the framework will generate scroll momentum.
656  NSAssert(event.momentumPhase != NSEventPhaseNone,
657  @"Received gesture event with unexpected phase");
658  }
659 }
660 
661 - (void)dispatchMouseEvent:(NSEvent*)event phase:(FlutterPointerPhase)phase {
662  NSAssert(self.viewLoaded, @"View must be loaded before it handles the mouse event");
663  // There are edge cases where the system will deliver enter out of order relative to other
664  // events (e.g., drag out and back in, release, then click; mouseDown: will be called before
665  // mouseEntered:). Discard those events, since the add will already have been synthesized.
666  if (_mouseState.flutter_state_is_added && phase == kAdd) {
667  return;
668  }
669 
670  // Multiple gesture recognizers could be active at once, we can't send multiple kPanZoomStart.
671  // For example: rotation and magnification.
672  if (phase == kPanZoomStart || phase == kPanZoomEnd) {
673  if (event.type == NSEventTypeScrollWheel) {
674  _mouseState.pan_gesture_phase = event.phase;
675  } else if (event.type == NSEventTypeMagnify) {
676  _mouseState.scale_gesture_phase = event.phase;
677  } else if (event.type == NSEventTypeRotate) {
678  _mouseState.rotate_gesture_phase = event.phase;
679  }
680  }
681  if (phase == kPanZoomStart) {
682  if (event.type == NSEventTypeScrollWheel) {
683  // Ensure scroll inertia cancel event is not sent afterwards.
684  _mouseState.last_scroll_momentum_changed_time = 0;
685  }
686  if (_mouseState.flutter_state_is_pan_zoom_started) {
687  // Already started on a previous gesture type
688  return;
689  }
690  _mouseState.flutter_state_is_pan_zoom_started = true;
691  }
692  if (phase == kPanZoomEnd) {
693  if (!_mouseState.flutter_state_is_pan_zoom_started) {
694  // NSEventPhaseCancelled is sometimes received at incorrect times in the state
695  // machine, just ignore it here if it doesn't make sense
696  // (we have no active gesture to cancel).
697  NSAssert(event.phase == NSEventPhaseCancelled,
698  @"Received gesture event with unexpected phase");
699  return;
700  }
701  // NSEventPhase values are powers of two, we can use this to inspect merged phases.
702  NSEventPhase all_gestures_fields = _mouseState.pan_gesture_phase |
703  _mouseState.scale_gesture_phase |
704  _mouseState.rotate_gesture_phase;
705  NSEventPhase active_mask = NSEventPhaseBegan | NSEventPhaseChanged;
706  if ((all_gestures_fields & active_mask) != 0) {
707  // Even though this gesture type ended, a different type is still active.
708  return;
709  }
710  }
711 
712  // If a pointer added event hasn't been sent, synthesize one using this event for the basic
713  // information.
714  if (!_mouseState.flutter_state_is_added && phase != kAdd) {
715  // Only the values extracted for use in flutterEvent below matter, the rest are dummy values.
716  NSEvent* addEvent = [NSEvent enterExitEventWithType:NSEventTypeMouseEntered
717  location:event.locationInWindow
718  modifierFlags:0
719  timestamp:event.timestamp
720  windowNumber:event.windowNumber
721  context:nil
722  eventNumber:0
723  trackingNumber:0
724  userData:NULL];
725  [self dispatchMouseEvent:addEvent phase:kAdd];
726  }
727 
728  NSPoint locationInView = [self.flutterView convertPoint:event.locationInWindow fromView:nil];
729  NSPoint locationInBackingCoordinates = [self.flutterView convertPointToBacking:locationInView];
730  int32_t device = kMousePointerDeviceId;
731  FlutterPointerDeviceKind deviceKind = kFlutterPointerDeviceKindMouse;
732  if (phase == kPanZoomStart || phase == kPanZoomUpdate || phase == kPanZoomEnd) {
733  device = kPointerPanZoomDeviceId;
734  deviceKind = kFlutterPointerDeviceKindTrackpad;
735  }
736  FlutterPointerEvent flutterEvent = {
737  .struct_size = sizeof(flutterEvent),
738  .phase = phase,
739  .timestamp = static_cast<size_t>(event.timestamp * USEC_PER_SEC),
740  .x = locationInBackingCoordinates.x,
741  .y = -locationInBackingCoordinates.y, // convertPointToBacking makes this negative.
742  .device = device,
743  .device_kind = deviceKind,
744  // If a click triggered a synthesized kAdd, don't pass the buttons in that event.
745  .buttons = phase == kAdd ? 0 : _mouseState.buttons,
746  .view_id = static_cast<FlutterViewId>(_viewId),
747  };
748 
749  if (phase == kPanZoomUpdate) {
750  if (event.type == NSEventTypeScrollWheel) {
751  _mouseState.delta_x += event.scrollingDeltaX * self.flutterView.layer.contentsScale;
752  _mouseState.delta_y += event.scrollingDeltaY * self.flutterView.layer.contentsScale;
753  } else if (event.type == NSEventTypeMagnify) {
754  _mouseState.scale += event.magnification;
755  } else if (event.type == NSEventTypeRotate) {
756  _mouseState.rotation += event.rotation * (-M_PI / 180.0);
757  }
758  flutterEvent.pan_x = _mouseState.delta_x;
759  flutterEvent.pan_y = _mouseState.delta_y;
760  // Scale value needs to be normalized to range 0->infinity.
761  flutterEvent.scale = pow(2.0, _mouseState.scale);
762  flutterEvent.rotation = _mouseState.rotation;
763  } else if (phase == kPanZoomEnd) {
764  _mouseState.GestureReset();
765  } else if (phase != kPanZoomStart && event.type == NSEventTypeScrollWheel) {
766  flutterEvent.signal_kind = kFlutterPointerSignalKindScroll;
767 
768  double pixelsPerLine = 1.0;
769  if (!event.hasPreciseScrollingDeltas) {
770  // The scrollingDelta needs to be multiplied by the line height.
771  // CGEventSourceGetPixelsPerLine() will return 10, which will result in
772  // scrolling that is noticeably slower than in other applications.
773  // Using 40.0 as the multiplier to match Chromium.
774  // See https://source.chromium.org/chromium/chromium/src/+/main:ui/events/cocoa/events_mac.mm
775  pixelsPerLine = 40.0;
776  }
777  double scaleFactor = self.flutterView.layer.contentsScale;
778  // When mouse input is received while shift is pressed (regardless of
779  // any other pressed keys), Mac automatically flips the axis. Other
780  // platforms do not do this, so we flip it back to normalize the input
781  // received by the framework. The keyboard+mouse-scroll mechanism is exposed
782  // in the ScrollBehavior of the framework so developers can customize the
783  // behavior.
784  // At time of change, Apple does not expose any other type of API or signal
785  // that the X/Y axes have been flipped.
786  double scaledDeltaX = -event.scrollingDeltaX * pixelsPerLine * scaleFactor;
787  double scaledDeltaY = -event.scrollingDeltaY * pixelsPerLine * scaleFactor;
788  if (event.modifierFlags & NSShiftKeyMask) {
789  flutterEvent.scroll_delta_x = scaledDeltaY;
790  flutterEvent.scroll_delta_y = scaledDeltaX;
791  } else {
792  flutterEvent.scroll_delta_x = scaledDeltaX;
793  flutterEvent.scroll_delta_y = scaledDeltaY;
794  }
795  }
796 
797  [_keyboardManager syncModifiersIfNeeded:event.modifierFlags timestamp:event.timestamp];
798  [_engine sendPointerEvent:flutterEvent];
799 
800  // Update tracking of state as reported to Flutter.
801  if (phase == kDown) {
802  _mouseState.flutter_state_is_down = true;
803  } else if (phase == kUp) {
804  _mouseState.flutter_state_is_down = false;
805  if (_mouseState.has_pending_exit) {
806  [self dispatchMouseEvent:event phase:kRemove];
807  _mouseState.has_pending_exit = false;
808  }
809  } else if (phase == kAdd) {
810  _mouseState.flutter_state_is_added = true;
811  } else if (phase == kRemove) {
812  _mouseState.Reset();
813  }
814 }
815 
816 - (void)onAccessibilityStatusChanged:(BOOL)enabled {
817  if (!enabled && self.viewLoaded && [_textInputPlugin isFirstResponder]) {
818  // Normally TextInputPlugin, when editing, is child of FlutterViewWrapper.
819  // When accessiblity is enabled the TextInputPlugin gets added as an indirect
820  // child to FlutterTextField. When disabling the plugin needs to be reparented
821  // back.
822  [self.view addSubview:_textInputPlugin];
823  }
824 }
825 
826 - (std::shared_ptr<flutter::AccessibilityBridgeMac>)createAccessibilityBridgeWithEngine:
827  (nonnull FlutterEngine*)engine {
828  return std::make_shared<flutter::AccessibilityBridgeMac>(engine, self);
829 }
830 
831 - (nonnull FlutterView*)createFlutterViewWithMTLDevice:(id<MTLDevice>)device
832  commandQueue:(id<MTLCommandQueue>)commandQueue {
833  return [[FlutterView alloc] initWithMTLDevice:device
834  commandQueue:commandQueue
835  delegate:self
836  threadSynchronizer:_threadSynchronizer
837  viewId:_viewId];
838 }
839 
840 - (void)onKeyboardLayoutChanged {
841  _keyboardLayoutData = nil;
842  if (_keyboardLayoutNotifier != nil) {
844  }
845 }
846 
847 - (NSString*)lookupKeyForAsset:(NSString*)asset {
848  return [FlutterDartProject lookupKeyForAsset:asset];
849 }
850 
851 - (NSString*)lookupKeyForAsset:(NSString*)asset fromPackage:(NSString*)package {
852  return [FlutterDartProject lookupKeyForAsset:asset fromPackage:package];
853 }
854 
855 #pragma mark - FlutterViewDelegate
856 
857 /**
858  * Responds to view reshape by notifying the engine of the change in dimensions.
859  */
860 - (void)viewDidReshape:(NSView*)view {
861  FML_DCHECK(view == _flutterView);
862  [_engine updateWindowMetricsForViewController:self];
863 }
864 
865 - (BOOL)viewShouldAcceptFirstResponder:(NSView*)view {
866  FML_DCHECK(view == _flutterView);
867  // Only allow FlutterView to become first responder if TextInputPlugin is
868  // not active. Otherwise a mouse event inside FlutterView would cause the
869  // TextInputPlugin to lose first responder status.
870  return !_textInputPlugin.isFirstResponder;
871 }
872 
873 #pragma mark - FlutterPluginRegistry
874 
875 - (id<FlutterPluginRegistrar>)registrarForPlugin:(NSString*)pluginName {
876  return [_engine registrarForPlugin:pluginName];
877 }
878 
879 - (NSObject*)valuePublishedByPlugin:(NSString*)pluginKey {
880  return [_engine valuePublishedByPlugin:pluginKey];
881 }
882 
883 #pragma mark - FlutterKeyboardViewDelegate
884 
885 /**
886  * Returns the current Unicode layout data (kTISPropertyUnicodeKeyLayoutData).
887  *
888  * To use the returned data, convert it to CFDataRef first, finds its bytes
889  * with CFDataGetBytePtr, then reinterpret it into const UCKeyboardLayout*.
890  * It's returned in NSData* to enable auto reference count.
891  */
892 static NSData* CurrentKeyboardLayoutData() {
893  TISInputSourceRef source = TISCopyCurrentKeyboardInputSource();
894  CFTypeRef layout_data = TISGetInputSourceProperty(source, kTISPropertyUnicodeKeyLayoutData);
895  if (layout_data == nil) {
896  CFRelease(source);
897  // TISGetInputSourceProperty returns null with Japanese keyboard layout.
898  // Using TISCopyCurrentKeyboardLayoutInputSource to fix NULL return.
899  // https://github.com/microsoft/node-native-keymap/blob/5f0699ded00179410a14c0e1b0e089fe4df8e130/src/keyboard_mac.mm#L91
900  source = TISCopyCurrentKeyboardLayoutInputSource();
901  layout_data = TISGetInputSourceProperty(source, kTISPropertyUnicodeKeyLayoutData);
902  }
903  return (__bridge_transfer NSData*)CFRetain(layout_data);
904 }
905 
906 - (void)sendKeyEvent:(const FlutterKeyEvent&)event
907  callback:(nullable FlutterKeyEventCallback)callback
908  userData:(nullable void*)userData {
909  [_engine sendKeyEvent:event callback:callback userData:userData];
910 }
911 
912 - (id<FlutterBinaryMessenger>)getBinaryMessenger {
913  return _engine.binaryMessenger;
914 }
915 
916 - (BOOL)onTextInputKeyEvent:(nonnull NSEvent*)event {
917  return [_textInputPlugin handleKeyEvent:event];
918 }
919 
920 - (void)subscribeToKeyboardLayoutChange:(nullable KeyboardLayoutNotifier)callback {
921  _keyboardLayoutNotifier = callback;
922 }
923 
924 - (LayoutClue)lookUpLayoutForKeyCode:(uint16_t)keyCode shift:(BOOL)shift {
925  if (_keyboardLayoutData == nil) {
926  _keyboardLayoutData = CurrentKeyboardLayoutData();
927  }
928  const UCKeyboardLayout* layout = reinterpret_cast<const UCKeyboardLayout*>(
929  CFDataGetBytePtr((__bridge CFDataRef)_keyboardLayoutData));
930 
931  UInt32 deadKeyState = 0;
932  UniCharCount stringLength = 0;
933  UniChar resultChar;
934 
935  UInt32 modifierState = ((shift ? shiftKey : 0) >> 8) & 0xFF;
936  UInt32 keyboardType = LMGetKbdLast();
937 
938  bool isDeadKey = false;
939  OSStatus status =
940  UCKeyTranslate(layout, keyCode, kUCKeyActionDown, modifierState, keyboardType,
941  kUCKeyTranslateNoDeadKeysBit, &deadKeyState, 1, &stringLength, &resultChar);
942  // For dead keys, press the same key again to get the printable representation of the key.
943  if (status == noErr && stringLength == 0 && deadKeyState != 0) {
944  isDeadKey = true;
945  status =
946  UCKeyTranslate(layout, keyCode, kUCKeyActionDown, modifierState, keyboardType,
947  kUCKeyTranslateNoDeadKeysBit, &deadKeyState, 1, &stringLength, &resultChar);
948  }
949 
950  if (status == noErr && stringLength == 1 && !std::iscntrl(resultChar)) {
951  return LayoutClue{resultChar, isDeadKey};
952  }
953  return LayoutClue{0, false};
954 }
955 
956 - (nonnull NSDictionary*)getPressedState {
957  return [_keyboardManager getPressedState];
958 }
959 
960 #pragma mark - NSResponder
961 
962 - (BOOL)acceptsFirstResponder {
963  return YES;
964 }
965 
966 - (void)keyDown:(NSEvent*)event {
967  [_keyboardManager handleEvent:event];
968 }
969 
970 - (void)keyUp:(NSEvent*)event {
971  [_keyboardManager handleEvent:event];
972 }
973 
974 - (void)flagsChanged:(NSEvent*)event {
975  [_keyboardManager handleEvent:event];
976 }
977 
978 - (void)mouseEntered:(NSEvent*)event {
979  if (_mouseState.has_pending_exit) {
980  _mouseState.has_pending_exit = false;
981  } else {
982  [self dispatchMouseEvent:event phase:kAdd];
983  }
984 }
985 
986 - (void)mouseExited:(NSEvent*)event {
987  if (_mouseState.buttons != 0) {
988  _mouseState.has_pending_exit = true;
989  return;
990  }
991  [self dispatchMouseEvent:event phase:kRemove];
992 }
993 
994 - (void)mouseDown:(NSEvent*)event {
995  _mouseState.buttons |= kFlutterPointerButtonMousePrimary;
996  [self dispatchMouseEvent:event];
997 }
998 
999 - (void)mouseUp:(NSEvent*)event {
1000  _mouseState.buttons &= ~static_cast<uint64_t>(kFlutterPointerButtonMousePrimary);
1001  [self dispatchMouseEvent:event];
1002 }
1003 
1004 - (void)mouseDragged:(NSEvent*)event {
1005  [self dispatchMouseEvent:event];
1006 }
1007 
1008 - (void)rightMouseDown:(NSEvent*)event {
1009  _mouseState.buttons |= kFlutterPointerButtonMouseSecondary;
1010  [self dispatchMouseEvent:event];
1011 }
1012 
1013 - (void)rightMouseUp:(NSEvent*)event {
1014  _mouseState.buttons &= ~static_cast<uint64_t>(kFlutterPointerButtonMouseSecondary);
1015  [self dispatchMouseEvent:event];
1016 }
1017 
1018 - (void)rightMouseDragged:(NSEvent*)event {
1019  [self dispatchMouseEvent:event];
1020 }
1021 
1022 - (void)otherMouseDown:(NSEvent*)event {
1023  _mouseState.buttons |= (1 << event.buttonNumber);
1024  [self dispatchMouseEvent:event];
1025 }
1026 
1027 - (void)otherMouseUp:(NSEvent*)event {
1028  _mouseState.buttons &= ~static_cast<uint64_t>(1 << event.buttonNumber);
1029  [self dispatchMouseEvent:event];
1030 }
1031 
1032 - (void)otherMouseDragged:(NSEvent*)event {
1033  [self dispatchMouseEvent:event];
1034 }
1035 
1036 - (void)mouseMoved:(NSEvent*)event {
1037  [self dispatchMouseEvent:event];
1038 }
1039 
1040 - (void)scrollWheel:(NSEvent*)event {
1041  [self dispatchGestureEvent:event];
1042 }
1043 
1044 - (void)magnifyWithEvent:(NSEvent*)event {
1045  [self dispatchGestureEvent:event];
1046 }
1047 
1048 - (void)rotateWithEvent:(NSEvent*)event {
1049  [self dispatchGestureEvent:event];
1050 }
1051 
1052 - (void)swipeWithEvent:(NSEvent*)event {
1053  // Not needed, it's handled by scrollWheel.
1054 }
1055 
1056 - (void)touchesBeganWithEvent:(NSEvent*)event {
1057  NSTouch* touch = event.allTouches.anyObject;
1058  if (touch != nil) {
1059  if ((event.timestamp - _mouseState.last_scroll_momentum_changed_time) <
1060  kTrackpadTouchInertiaCancelWindowMs) {
1061  // The trackpad has been touched following a scroll momentum event.
1062  // A scroll inertia cancel message should be sent to the framework.
1063  NSPoint locationInView = [self.flutterView convertPoint:event.locationInWindow fromView:nil];
1064  NSPoint locationInBackingCoordinates =
1065  [self.flutterView convertPointToBacking:locationInView];
1066  FlutterPointerEvent flutterEvent = {
1067  .struct_size = sizeof(flutterEvent),
1068  .timestamp = static_cast<size_t>(event.timestamp * USEC_PER_SEC),
1069  .x = locationInBackingCoordinates.x,
1070  .y = -locationInBackingCoordinates.y, // convertPointToBacking makes this negative.
1071  .device = kPointerPanZoomDeviceId,
1072  .signal_kind = kFlutterPointerSignalKindScrollInertiaCancel,
1073  .device_kind = kFlutterPointerDeviceKindTrackpad,
1074  .view_id = static_cast<FlutterViewId>(_viewId),
1075  };
1076 
1077  [_engine sendPointerEvent:flutterEvent];
1078  // Ensure no further scroll inertia cancel event will be sent.
1079  _mouseState.last_scroll_momentum_changed_time = 0;
1080  }
1081  }
1082 }
1083 
1084 @end
flutter::LayoutClue
Definition: FlutterKeyboardViewDelegate.h:20
FlutterEngine
Definition: FlutterEngine.h:30
FlutterViewController
Definition: FlutterViewController.h:65
FlutterEngine.h
FlutterViewWrapper
Definition: FlutterViewController.mm:153
FlutterEngine_Internal.h
+[FlutterDartProject lookupKeyForAsset:]
NSString * lookupKeyForAsset:(NSString *asset)
Definition: FlutterDartProject.mm:116
_keyboardLayoutNotifier
flutter::KeyboardLayoutNotifier _keyboardLayoutNotifier
Definition: FlutterKeyboardManagerTest.mm:243
_bridge
std::shared_ptr< flutter::AccessibilityBridgeMac > _bridge
Definition: FlutterViewController.mm:320
FlutterChannels.h
FlutterRenderer.h
_project
FlutterDartProject * _project
Definition: FlutterEngine.mm:403
FlutterViewController::engine
FlutterEngine * engine
Definition: FlutterViewController.h:70
FlutterPluginRegistrar-p
Definition: FlutterPluginRegistrarMacOS.h:28
FlutterKeyPrimaryResponder.h
-[FlutterView setBackgroundColor:]
void setBackgroundColor:(nonnull NSColor *color)
OnKeyboardLayoutChanged
static void OnKeyboardLayoutChanged(CFNotificationCenterRef center, void *observer, CFStringRef name, const void *object, CFDictionaryRef userInfo)
Definition: FlutterViewController.mm:238
_controller
__weak FlutterViewController * _controller
Definition: FlutterViewController.mm:249
_id
FlutterViewId _id
Definition: FlutterViewController.mm:326
FlutterThreadSynchronizer
Definition: FlutterThreadSynchronizer.h:16
flutter::KeyboardLayoutNotifier
void(^ KeyboardLayoutNotifier)()
Definition: FlutterKeyboardViewDelegate.h:16
FlutterTextInputPlugin
Definition: FlutterTextInputPlugin.h:27
FlutterCodecs.h
FlutterKeyboardManager.h
FlutterViewController_Internal.h
_threadSynchronizer
FlutterThreadSynchronizer * _threadSynchronizer
Definition: FlutterViewController.mm:330
FlutterView
Definition: FlutterView.h:48
FlutterTextInputSemanticsObject.h
+[FlutterDartProject lookupKeyForAsset:fromPackage:]
NSString * lookupKeyForAsset:fromPackage:(NSString *asset,[fromPackage] NSString *package)
Definition: FlutterDartProject.mm:125
FlutterDartProject
Definition: FlutterDartProject.mm:24
FlutterKeyboardManager
Definition: FlutterKeyboardManager.h:27
FlutterBinaryMessenger-p
Definition: FlutterBinaryMessenger.h:49
FlutterView.h
FlutterViewDelegate-p
Definition: FlutterView.h:31
FlutterViewController.h
FlutterViewId
int64_t FlutterViewId
Definition: FlutterView.h:15