Flutter macOS Embedder
FlutterViewController.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
7 
8 #include <Carbon/Carbon.h>
9 #import <objc/message.h>
10 
11 #include "flutter/common/constants.h"
12 #include "flutter/fml/platform/darwin/cf_utils.h"
22 #include "flutter/shell/platform/embedder/embedder.h"
23 
24 #pragma mark - Static types and data.
25 
26 namespace {
29 
30 // Use different device ID for mouse and pan/zoom events, since we can't differentiate the actual
31 // device (mouse v.s. trackpad).
32 static constexpr int32_t kMousePointerDeviceId = 0;
33 static constexpr int32_t kPointerPanZoomDeviceId = 1;
34 
35 // A trackpad touch following inertial scrolling should cause an inertia cancel
36 // event to be issued. Use a window of 50 milliseconds after the scroll to account
37 // for delays in event propagation observed in macOS Ventura.
38 static constexpr double kTrackpadTouchInertiaCancelWindowMs = 0.050;
39 
40 /**
41  * State tracking for mouse events, to adapt between the events coming from the system and the
42  * events that the embedding API expects.
43  */
44 struct MouseState {
45  /**
46  * The currently pressed buttons, as represented in FlutterPointerEvent.
47  */
48  int64_t buttons = 0;
49 
50  /**
51  * The accumulated gesture pan.
52  */
53  CGFloat delta_x = 0;
54  CGFloat delta_y = 0;
55 
56  /**
57  * The accumulated gesture zoom scale.
58  */
59  CGFloat scale = 0;
60 
61  /**
62  * The accumulated gesture rotation.
63  */
64  CGFloat rotation = 0;
65 
66  /**
67  * Whether or not a kAdd event has been sent (or sent again since the last kRemove if tracking is
68  * enabled). Used to determine whether to send a kAdd event before sending an incoming mouse
69  * event, since Flutter expects pointers to be added before events are sent for them.
70  */
71  bool flutter_state_is_added = false;
72 
73  /**
74  * Whether or not a kDown has been sent since the last kAdd/kUp.
75  */
76  bool flutter_state_is_down = false;
77 
78  /**
79  * Whether or not mouseExited: was received while a button was down. Cocoa's behavior when
80  * dragging out of a tracked area is to send an exit, then keep sending drag events until the last
81  * button is released. Flutter doesn't expect to receive events after a kRemove, so the kRemove
82  * for the exit needs to be delayed until after the last mouse button is released. If cursor
83  * returns back to the window while still dragging, the flag is cleared in mouseEntered:.
84  */
85  bool has_pending_exit = false;
86 
87  /*
88  * Whether or not a kPanZoomStart has been sent since the last kAdd/kPanZoomEnd.
89  */
90  bool flutter_state_is_pan_zoom_started = false;
91 
92  /**
93  * State of pan gesture.
94  */
95  NSEventPhase pan_gesture_phase = NSEventPhaseNone;
96 
97  /**
98  * State of scale gesture.
99  */
100  NSEventPhase scale_gesture_phase = NSEventPhaseNone;
101 
102  /**
103  * State of rotate gesture.
104  */
105  NSEventPhase rotate_gesture_phase = NSEventPhaseNone;
106 
107  /**
108  * Time of last scroll momentum event.
109  */
110  NSTimeInterval last_scroll_momentum_changed_time = 0;
111 
112  /**
113  * Resets all gesture state to default values.
114  */
115  void GestureReset() {
116  delta_x = 0;
117  delta_y = 0;
118  scale = 0;
119  rotation = 0;
120  flutter_state_is_pan_zoom_started = false;
121  pan_gesture_phase = NSEventPhaseNone;
122  scale_gesture_phase = NSEventPhaseNone;
123  rotate_gesture_phase = NSEventPhaseNone;
124  }
125 
126  /**
127  * Resets all state to default values.
128  */
129  void Reset() {
130  flutter_state_is_added = false;
131  flutter_state_is_down = false;
132  has_pending_exit = false;
133  buttons = 0;
134  }
135 };
136 
137 } // namespace
138 
139 #pragma mark - Private interface declaration.
140 
141 /**
142  * FlutterViewWrapper is a convenience class that wraps a FlutterView and provides
143  * a mechanism to attach AppKit views such as FlutterTextField without affecting
144  * the accessibility subtree of the wrapped FlutterView itself.
145  *
146  * The FlutterViewController uses this class to create its content view. When
147  * any of the accessibility services (e.g. VoiceOver) is turned on, the accessibility
148  * bridge creates FlutterTextFields that interact with the service. The bridge has to
149  * attach the FlutterTextField somewhere in the view hierarchy in order for the
150  * FlutterTextField to interact correctly with VoiceOver. Those FlutterTextFields
151  * will be attached to this view so that they won't affect the accessibility subtree
152  * of FlutterView.
153  */
154 @interface FlutterViewWrapper : NSView
155 
156 - (void)setBackgroundColor:(NSColor*)color;
157 
158 @end
159 
160 /**
161  * Private interface declaration for FlutterViewController.
162  */
164 
165 /**
166  * The tracking area used to generate hover events, if enabled.
167  */
168 @property(nonatomic) NSTrackingArea* trackingArea;
169 
170 /**
171  * The current state of the mouse and the sent mouse events.
172  */
173 @property(nonatomic) MouseState mouseState;
174 
175 /**
176  * Event monitor for keyUp events.
177  */
178 @property(nonatomic) id keyUpMonitor;
179 
180 /**
181  * Pointer to a keyboard manager, a hub that manages how key events are
182  * dispatched to various Flutter key responders, and whether the event is
183  * propagated to the next NSResponder.
184  */
185 @property(nonatomic, readonly, nonnull) FlutterKeyboardManager* keyboardManager;
186 
187 @property(nonatomic) KeyboardLayoutNotifier keyboardLayoutNotifier;
188 
189 @property(nonatomic) NSData* keyboardLayoutData;
190 
191 /**
192  * Starts running |engine|, including any initial setup.
193  */
194 - (BOOL)launchEngine;
195 
196 /**
197  * Updates |trackingArea| for the current tracking settings, creating it with
198  * the correct mode if tracking is enabled, or removing it if not.
199  */
200 - (void)configureTrackingArea;
201 
202 /**
203  * Creates and registers keyboard related components.
204  */
205 - (void)initializeKeyboard;
206 
207 /**
208  * Calls dispatchMouseEvent:phase: with a phase determined by self.mouseState.
209  *
210  * mouseState.buttons should be updated before calling this method.
211  */
212 - (void)dispatchMouseEvent:(nonnull NSEvent*)event;
213 
214 /**
215  * Calls dispatchMouseEvent:phase: with a phase determined by event.phase.
216  */
217 - (void)dispatchGestureEvent:(nonnull NSEvent*)event;
218 
219 /**
220  * Converts |event| to a FlutterPointerEvent with the given phase, and sends it to the engine.
221  */
222 - (void)dispatchMouseEvent:(nonnull NSEvent*)event phase:(FlutterPointerPhase)phase;
223 
224 /**
225  * Called when the active keyboard input source changes.
226  *
227  * Input sources may be simple keyboard layouts, or more complex input methods involving an IME,
228  * such as Chinese, Japanese, and Korean.
229  */
230 - (void)onKeyboardLayoutChanged;
231 
232 @end
233 
234 #pragma mark - FlutterViewWrapper implementation.
235 
236 /**
237  * NotificationCenter callback invoked on kTISNotifySelectedKeyboardInputSourceChanged events.
238  */
239 static void OnKeyboardLayoutChanged(CFNotificationCenterRef center,
240  void* observer,
241  CFStringRef name,
242  const void* object,
243  CFDictionaryRef userInfo) {
244  FlutterViewController* controller = (__bridge FlutterViewController*)observer;
245  if (controller != nil) {
246  [controller onKeyboardLayoutChanged];
247  }
248 }
249 
250 @implementation FlutterViewWrapper {
251  FlutterView* _flutterView;
253 }
254 
255 - (instancetype)initWithFlutterView:(FlutterView*)view
256  controller:(FlutterViewController*)controller {
257  self = [super initWithFrame:NSZeroRect];
258  if (self) {
259  _flutterView = view;
260  _controller = controller;
261  view.autoresizingMask = NSViewWidthSizable | NSViewHeightSizable;
262  [self addSubview:view];
263  }
264  return self;
265 }
266 
267 - (void)setBackgroundColor:(NSColor*)color {
268  [_flutterView setBackgroundColor:color];
269 }
270 
271 - (BOOL)performKeyEquivalent:(NSEvent*)event {
272  // Do not intercept the event if flutterView is not first responder, otherwise this would
273  // interfere with TextInputPlugin, which also handles key equivalents.
274  //
275  // Also do not intercept the event if key equivalent is a product of an event being
276  // redispatched by the TextInputPlugin, in which case it needs to bubble up so that menus
277  // can handle key equivalents.
278  if (self.window.firstResponder != _flutterView || [_controller isDispatchingKeyEvent:event]) {
279  return [super performKeyEquivalent:event];
280  }
281  [_flutterView keyDown:event];
282  return YES;
283 }
284 
285 - (NSArray*)accessibilityChildren {
286  return @[ _flutterView ];
287 }
288 
289 // TODO(cbracken): https://github.com/flutter/flutter/issues/154063
290 // Remove this whole method override when we drop support for macOS 12 (Monterey).
291 - (void)mouseDown:(NSEvent*)event {
292  if (@available(macOS 13.3.1, *)) {
293  [super mouseDown:event];
294  } else {
295  // Work around an AppKit bug where mouseDown/mouseUp are not called on the view controller if
296  // the view is the content view of an NSPopover AND macOS's Reduced Transparency accessibility
297  // setting is enabled.
298  //
299  // This simply calls mouseDown on the next responder in the responder chain as the default
300  // implementation on NSResponder is documented to do.
301  //
302  // See: https://github.com/flutter/flutter/issues/115015
303  // See: http://www.openradar.me/FB12050037
304  // See: https://developer.apple.com/documentation/appkit/nsresponder/1524634-mousedown
305  [self.nextResponder mouseDown:event];
306  }
307 }
308 
309 // TODO(cbracken): https://github.com/flutter/flutter/issues/154063
310 // Remove this workaround when we drop support for macOS 12 (Monterey).
311 - (void)mouseUp:(NSEvent*)event {
312  if (@available(macOS 13.3.1, *)) {
313  [super mouseUp:event];
314  } else {
315  // Work around an AppKit bug where mouseDown/mouseUp are not called on the view controller if
316  // the view is the content view of an NSPopover AND macOS's Reduced Transparency accessibility
317  // setting is enabled.
318  //
319  // This simply calls mouseUp on the next responder in the responder chain as the default
320  // implementation on NSResponder is documented to do.
321  //
322  // See: https://github.com/flutter/flutter/issues/115015
323  // See: http://www.openradar.me/FB12050037
324  // See: https://developer.apple.com/documentation/appkit/nsresponder/1535349-mouseup
325  [self.nextResponder mouseUp:event];
326  }
327 }
328 
329 @end
330 
331 #pragma mark - FlutterViewController implementation.
332 
333 @implementation FlutterViewController {
334  // The project to run in this controller's engine.
336 
337  std::shared_ptr<flutter::AccessibilityBridgeMac> _bridge;
338 
339  // FlutterViewController does not actually uses the synchronizer, but only
340  // passes it to FlutterView.
342 }
343 
344 // Synthesize properties declared readonly.
345 @synthesize viewIdentifier = _viewIdentifier;
346 
347 @dynamic accessibilityBridge;
348 
349 /**
350  * Performs initialization that's common between the different init paths.
351  */
352 static void CommonInit(FlutterViewController* controller, FlutterEngine* engine) {
353  if (!engine) {
354  engine = [[FlutterEngine alloc] initWithName:@"io.flutter"
355  project:controller->_project
356  allowHeadlessExecution:NO];
357  }
358  NSCAssert(controller.engine == nil,
359  @"The FlutterViewController is unexpectedly attached to "
360  @"engine %@ before initialization.",
361  controller.engine);
362  [engine addViewController:controller];
363  NSCAssert(controller.engine != nil,
364  @"The FlutterViewController unexpectedly stays unattached after initialization. "
365  @"In unit tests, this is likely because either the FlutterViewController or "
366  @"the FlutterEngine is mocked. Please subclass these classes instead.",
367  controller.engine, controller.viewIdentifier);
368  controller->_mouseTrackingMode = kFlutterMouseTrackingModeInKeyWindow;
369  controller->_textInputPlugin = [[FlutterTextInputPlugin alloc] initWithViewController:controller];
370  [controller initializeKeyboard];
371  [controller notifySemanticsEnabledChanged];
372  // macOS fires this message when changing IMEs.
373  CFNotificationCenterRef cfCenter = CFNotificationCenterGetDistributedCenter();
374  __weak FlutterViewController* weakSelf = controller;
375  CFNotificationCenterAddObserver(cfCenter, (__bridge void*)weakSelf, OnKeyboardLayoutChanged,
376  kTISNotifySelectedKeyboardInputSourceChanged, NULL,
377  CFNotificationSuspensionBehaviorDeliverImmediately);
378 }
379 
380 - (instancetype)initWithCoder:(NSCoder*)coder {
381  self = [super initWithCoder:coder];
382  NSAssert(self, @"Super init cannot be nil");
383 
384  CommonInit(self, nil);
385  return self;
386 }
387 
388 - (instancetype)initWithNibName:(NSString*)nibNameOrNil bundle:(NSBundle*)nibBundleOrNil {
389  self = [super initWithNibName:nibNameOrNil bundle:nibBundleOrNil];
390  NSAssert(self, @"Super init cannot be nil");
391 
392  CommonInit(self, nil);
393  return self;
394 }
395 
396 - (instancetype)initWithProject:(nullable FlutterDartProject*)project {
397  self = [super initWithNibName:nil bundle:nil];
398  NSAssert(self, @"Super init cannot be nil");
399 
400  _project = project;
401  CommonInit(self, nil);
402  return self;
403 }
404 
405 - (instancetype)initWithEngine:(nonnull FlutterEngine*)engine
406  nibName:(nullable NSString*)nibName
407  bundle:(nullable NSBundle*)nibBundle {
408  NSAssert(engine != nil, @"Engine is required");
409 
410  self = [super initWithNibName:nibName bundle:nibBundle];
411  if (self) {
412  CommonInit(self, engine);
413  }
414 
415  return self;
416 }
417 
418 - (BOOL)isDispatchingKeyEvent:(NSEvent*)event {
419  return [_keyboardManager isDispatchingKeyEvent:event];
420 }
421 
422 - (void)loadView {
423  FlutterView* flutterView;
424  id<MTLDevice> device = _engine.renderer.device;
425  id<MTLCommandQueue> commandQueue = _engine.renderer.commandQueue;
426  if (!device || !commandQueue) {
427  NSLog(@"Unable to create FlutterView; no MTLDevice or MTLCommandQueue available.");
428  return;
429  }
430  flutterView = [self createFlutterViewWithMTLDevice:device commandQueue:commandQueue];
431  if (_backgroundColor != nil) {
432  [flutterView setBackgroundColor:_backgroundColor];
433  }
434  FlutterViewWrapper* wrapperView = [[FlutterViewWrapper alloc] initWithFlutterView:flutterView
435  controller:self];
436  self.view = wrapperView;
437  _flutterView = flutterView;
438 }
439 
440 - (void)viewDidLoad {
441  [self configureTrackingArea];
442  [self.view setAllowedTouchTypes:NSTouchTypeMaskIndirect];
443  [self.view setWantsRestingTouches:YES];
444  [_engine viewControllerViewDidLoad:self];
445 }
446 
447 - (void)viewWillAppear {
448  [super viewWillAppear];
449  if (!_engine.running) {
450  [self launchEngine];
451  }
452  [self listenForMetaModifiedKeyUpEvents];
453 }
454 
455 - (void)viewWillDisappear {
456  // Per Apple's documentation, it is discouraged to call removeMonitor: in dealloc, and it's
457  // recommended to be called earlier in the lifecycle.
458  [NSEvent removeMonitor:_keyUpMonitor];
459  _keyUpMonitor = nil;
460 }
461 
462 - (void)dealloc {
463  if ([self attached]) {
464  [_engine removeViewController:self];
465  }
466  CFNotificationCenterRef cfCenter = CFNotificationCenterGetDistributedCenter();
467  CFNotificationCenterRemoveEveryObserver(cfCenter, (__bridge void*)self);
468 }
469 
470 #pragma mark - Public methods
471 
472 - (void)setMouseTrackingMode:(FlutterMouseTrackingMode)mode {
473  if (_mouseTrackingMode == mode) {
474  return;
475  }
476  _mouseTrackingMode = mode;
477  [self configureTrackingArea];
478 }
479 
480 - (void)setBackgroundColor:(NSColor*)color {
481  _backgroundColor = color;
482  [_flutterView setBackgroundColor:_backgroundColor];
483 }
484 
485 - (FlutterViewIdentifier)viewIdentifier {
486  NSAssert([self attached], @"This view controller is not attached.");
487  return _viewIdentifier;
488 }
489 
490 - (void)onPreEngineRestart {
491  [self initializeKeyboard];
492 }
493 
494 - (void)notifySemanticsEnabledChanged {
495  BOOL mySemanticsEnabled = !!_bridge;
496  BOOL newSemanticsEnabled = _engine.semanticsEnabled;
497  if (newSemanticsEnabled == mySemanticsEnabled) {
498  return;
499  }
500  if (newSemanticsEnabled) {
501  _bridge = [self createAccessibilityBridgeWithEngine:_engine];
502  } else {
503  // Remove the accessibility children from flutter view before resetting the bridge.
504  _flutterView.accessibilityChildren = nil;
505  _bridge.reset();
506  }
507  NSAssert(newSemanticsEnabled == !!_bridge, @"Failed to update semantics for the view.");
508 }
509 
510 - (std::weak_ptr<flutter::AccessibilityBridgeMac>)accessibilityBridge {
511  return _bridge;
512 }
513 
514 - (void)setUpWithEngine:(FlutterEngine*)engine
515  viewIdentifier:(FlutterViewIdentifier)viewIdentifier
516  threadSynchronizer:(FlutterThreadSynchronizer*)threadSynchronizer {
517  NSAssert(_engine == nil, @"Already attached to an engine %@.", _engine);
518  _engine = engine;
519  _viewIdentifier = viewIdentifier;
520  _threadSynchronizer = threadSynchronizer;
521  [_threadSynchronizer registerView:_viewIdentifier];
522 }
523 
524 - (void)detachFromEngine {
525  NSAssert(_engine != nil, @"Not attached to any engine.");
526  [_threadSynchronizer deregisterView:_viewIdentifier];
527  _threadSynchronizer = nil;
528  _engine = nil;
529 }
530 
531 - (BOOL)attached {
532  return _engine != nil;
533 }
534 
535 - (void)updateSemantics:(const FlutterSemanticsUpdate2*)update {
536  // Semantics will be disabled when unfocusing application but the updateSemantics:
537  // callback is received in next run loop turn.
538  if (!_engine.semanticsEnabled) {
539  return;
540  }
541  for (size_t i = 0; i < update->node_count; i++) {
542  const FlutterSemanticsNode2* node = update->nodes[i];
543  _bridge->AddFlutterSemanticsNodeUpdate(*node);
544  }
545 
546  for (size_t i = 0; i < update->custom_action_count; i++) {
547  const FlutterSemanticsCustomAction2* action = update->custom_actions[i];
548  _bridge->AddFlutterSemanticsCustomActionUpdate(*action);
549  }
550 
551  _bridge->CommitUpdates();
552 
553  // Accessibility tree can only be used when the view is loaded.
554  if (!self.viewLoaded) {
555  return;
556  }
557  // Attaches the accessibility root to the flutter view.
558  auto root = _bridge->GetFlutterPlatformNodeDelegateFromID(0).lock();
559  if (root) {
560  if ([self.flutterView.accessibilityChildren count] == 0) {
561  NSAccessibilityElement* native_root = root->GetNativeViewAccessible();
562  self.flutterView.accessibilityChildren = @[ native_root ];
563  }
564  } else {
565  self.flutterView.accessibilityChildren = nil;
566  }
567 }
568 
569 #pragma mark - Private methods
570 
571 - (BOOL)launchEngine {
572  if (![_engine runWithEntrypoint:nil]) {
573  return NO;
574  }
575  return YES;
576 }
577 
578 // macOS does not call keyUp: on a key while the command key is pressed. This results in a loss
579 // of a key event once the modified key is released. This method registers the
580 // ViewController as a listener for a keyUp event before it's handled by NSApplication, and should
581 // NOT modify the event to avoid any unexpected behavior.
582 - (void)listenForMetaModifiedKeyUpEvents {
583  if (_keyUpMonitor != nil) {
584  // It is possible for [NSViewController viewWillAppear] to be invoked multiple times
585  // in a row. https://github.com/flutter/flutter/issues/105963
586  return;
587  }
588  FlutterViewController* __weak weakSelf = self;
589  _keyUpMonitor = [NSEvent
590  addLocalMonitorForEventsMatchingMask:NSEventMaskKeyUp
591  handler:^NSEvent*(NSEvent* event) {
592  // Intercept keyUp only for events triggered on the current
593  // view or textInputPlugin.
594  NSResponder* firstResponder = [[event window] firstResponder];
595  if (weakSelf.viewLoaded && weakSelf.flutterView &&
596  (firstResponder == weakSelf.flutterView ||
597  firstResponder == weakSelf.textInputPlugin) &&
598  ([event modifierFlags] & NSEventModifierFlagCommand) &&
599  ([event type] == NSEventTypeKeyUp)) {
600  [weakSelf keyUp:event];
601  }
602  return event;
603  }];
604 }
605 
606 - (void)configureTrackingArea {
607  if (!self.viewLoaded) {
608  // The viewDidLoad will call configureTrackingArea again when
609  // the view is actually loaded.
610  return;
611  }
612  if (_mouseTrackingMode != kFlutterMouseTrackingModeNone && self.flutterView) {
613  NSTrackingAreaOptions options = NSTrackingMouseEnteredAndExited | NSTrackingMouseMoved |
614  NSTrackingInVisibleRect | NSTrackingEnabledDuringMouseDrag;
615  switch (_mouseTrackingMode) {
616  case kFlutterMouseTrackingModeInKeyWindow:
617  options |= NSTrackingActiveInKeyWindow;
618  break;
619  case kFlutterMouseTrackingModeInActiveApp:
620  options |= NSTrackingActiveInActiveApp;
621  break;
622  case kFlutterMouseTrackingModeAlways:
623  options |= NSTrackingActiveAlways;
624  break;
625  default:
626  NSLog(@"Error: Unrecognized mouse tracking mode: %ld", _mouseTrackingMode);
627  return;
628  }
629  _trackingArea = [[NSTrackingArea alloc] initWithRect:NSZeroRect
630  options:options
631  owner:self
632  userInfo:nil];
633  [self.flutterView addTrackingArea:_trackingArea];
634  } else if (_trackingArea) {
635  [self.flutterView removeTrackingArea:_trackingArea];
636  _trackingArea = nil;
637  }
638 }
639 
640 - (void)initializeKeyboard {
641  // TODO(goderbauer): Seperate keyboard/textinput stuff into ViewController specific and Engine
642  // global parts. Move the global parts to FlutterEngine.
643  _keyboardManager = [[FlutterKeyboardManager alloc] initWithViewDelegate:self];
644 }
645 
646 - (void)dispatchMouseEvent:(nonnull NSEvent*)event {
647  FlutterPointerPhase phase = _mouseState.buttons == 0
648  ? (_mouseState.flutter_state_is_down ? kUp : kHover)
649  : (_mouseState.flutter_state_is_down ? kMove : kDown);
650  [self dispatchMouseEvent:event phase:phase];
651 }
652 
653 - (void)dispatchGestureEvent:(nonnull NSEvent*)event {
654  if (event.phase == NSEventPhaseBegan || event.phase == NSEventPhaseMayBegin) {
655  [self dispatchMouseEvent:event phase:kPanZoomStart];
656  } else if (event.phase == NSEventPhaseChanged) {
657  [self dispatchMouseEvent:event phase:kPanZoomUpdate];
658  } else if (event.phase == NSEventPhaseEnded || event.phase == NSEventPhaseCancelled) {
659  [self dispatchMouseEvent:event phase:kPanZoomEnd];
660  } else if (event.phase == NSEventPhaseNone && event.momentumPhase == NSEventPhaseNone) {
661  [self dispatchMouseEvent:event phase:kHover];
662  } else {
663  // Waiting until the first momentum change event is a workaround for an issue where
664  // touchesBegan: is called unexpectedly while in low power mode within the interval between
665  // momentum start and the first momentum change.
666  if (event.momentumPhase == NSEventPhaseChanged) {
667  _mouseState.last_scroll_momentum_changed_time = event.timestamp;
668  }
669  // Skip momentum update events, the framework will generate scroll momentum.
670  NSAssert(event.momentumPhase != NSEventPhaseNone,
671  @"Received gesture event with unexpected phase");
672  }
673 }
674 
675 - (void)dispatchMouseEvent:(NSEvent*)event phase:(FlutterPointerPhase)phase {
676  NSAssert(self.viewLoaded, @"View must be loaded before it handles the mouse event");
677  // There are edge cases where the system will deliver enter out of order relative to other
678  // events (e.g., drag out and back in, release, then click; mouseDown: will be called before
679  // mouseEntered:). Discard those events, since the add will already have been synthesized.
680  if (_mouseState.flutter_state_is_added && phase == kAdd) {
681  return;
682  }
683 
684  // Multiple gesture recognizers could be active at once, we can't send multiple kPanZoomStart.
685  // For example: rotation and magnification.
686  if (phase == kPanZoomStart || phase == kPanZoomEnd) {
687  if (event.type == NSEventTypeScrollWheel) {
688  _mouseState.pan_gesture_phase = event.phase;
689  } else if (event.type == NSEventTypeMagnify) {
690  _mouseState.scale_gesture_phase = event.phase;
691  } else if (event.type == NSEventTypeRotate) {
692  _mouseState.rotate_gesture_phase = event.phase;
693  }
694  }
695  if (phase == kPanZoomStart) {
696  if (event.type == NSEventTypeScrollWheel) {
697  // Ensure scroll inertia cancel event is not sent afterwards.
698  _mouseState.last_scroll_momentum_changed_time = 0;
699  }
700  if (_mouseState.flutter_state_is_pan_zoom_started) {
701  // Already started on a previous gesture type
702  return;
703  }
704  _mouseState.flutter_state_is_pan_zoom_started = true;
705  }
706  if (phase == kPanZoomEnd) {
707  if (!_mouseState.flutter_state_is_pan_zoom_started) {
708  // NSEventPhaseCancelled is sometimes received at incorrect times in the state
709  // machine, just ignore it here if it doesn't make sense
710  // (we have no active gesture to cancel).
711  NSAssert(event.phase == NSEventPhaseCancelled,
712  @"Received gesture event with unexpected phase");
713  return;
714  }
715  // NSEventPhase values are powers of two, we can use this to inspect merged phases.
716  NSEventPhase all_gestures_fields = _mouseState.pan_gesture_phase |
717  _mouseState.scale_gesture_phase |
718  _mouseState.rotate_gesture_phase;
719  NSEventPhase active_mask = NSEventPhaseBegan | NSEventPhaseChanged;
720  if ((all_gestures_fields & active_mask) != 0) {
721  // Even though this gesture type ended, a different type is still active.
722  return;
723  }
724  }
725 
726  // If a pointer added event hasn't been sent, synthesize one using this event for the basic
727  // information.
728  if (!_mouseState.flutter_state_is_added && phase != kAdd) {
729  // Only the values extracted for use in flutterEvent below matter, the rest are dummy values.
730  NSEvent* addEvent = [NSEvent enterExitEventWithType:NSEventTypeMouseEntered
731  location:event.locationInWindow
732  modifierFlags:0
733  timestamp:event.timestamp
734  windowNumber:event.windowNumber
735  context:nil
736  eventNumber:0
737  trackingNumber:0
738  userData:NULL];
739  [self dispatchMouseEvent:addEvent phase:kAdd];
740  }
741 
742  NSPoint locationInView = [self.flutterView convertPoint:event.locationInWindow fromView:nil];
743  NSPoint locationInBackingCoordinates = [self.flutterView convertPointToBacking:locationInView];
744  int32_t device = kMousePointerDeviceId;
745  FlutterPointerDeviceKind deviceKind = kFlutterPointerDeviceKindMouse;
746  if (phase == kPanZoomStart || phase == kPanZoomUpdate || phase == kPanZoomEnd) {
747  device = kPointerPanZoomDeviceId;
748  deviceKind = kFlutterPointerDeviceKindTrackpad;
749  }
750  FlutterPointerEvent flutterEvent = {
751  .struct_size = sizeof(flutterEvent),
752  .phase = phase,
753  .timestamp = static_cast<size_t>(event.timestamp * USEC_PER_SEC),
754  .x = locationInBackingCoordinates.x,
755  .y = -locationInBackingCoordinates.y, // convertPointToBacking makes this negative.
756  .device = device,
757  .device_kind = deviceKind,
758  // If a click triggered a synthesized kAdd, don't pass the buttons in that event.
759  .buttons = phase == kAdd ? 0 : _mouseState.buttons,
760  .view_id = static_cast<FlutterViewIdentifier>(_viewIdentifier),
761  };
762 
763  if (phase == kPanZoomUpdate) {
764  if (event.type == NSEventTypeScrollWheel) {
765  _mouseState.delta_x += event.scrollingDeltaX * self.flutterView.layer.contentsScale;
766  _mouseState.delta_y += event.scrollingDeltaY * self.flutterView.layer.contentsScale;
767  } else if (event.type == NSEventTypeMagnify) {
768  _mouseState.scale += event.magnification;
769  } else if (event.type == NSEventTypeRotate) {
770  _mouseState.rotation += event.rotation * (-M_PI / 180.0);
771  }
772  flutterEvent.pan_x = _mouseState.delta_x;
773  flutterEvent.pan_y = _mouseState.delta_y;
774  // Scale value needs to be normalized to range 0->infinity.
775  flutterEvent.scale = pow(2.0, _mouseState.scale);
776  flutterEvent.rotation = _mouseState.rotation;
777  } else if (phase == kPanZoomEnd) {
778  _mouseState.GestureReset();
779  } else if (phase != kPanZoomStart && event.type == NSEventTypeScrollWheel) {
780  flutterEvent.signal_kind = kFlutterPointerSignalKindScroll;
781 
782  double pixelsPerLine = 1.0;
783  if (!event.hasPreciseScrollingDeltas) {
784  // The scrollingDelta needs to be multiplied by the line height.
785  // CGEventSourceGetPixelsPerLine() will return 10, which will result in
786  // scrolling that is noticeably slower than in other applications.
787  // Using 40.0 as the multiplier to match Chromium.
788  // See https://source.chromium.org/chromium/chromium/src/+/main:ui/events/cocoa/events_mac.mm
789  pixelsPerLine = 40.0;
790  }
791  double scaleFactor = self.flutterView.layer.contentsScale;
792  // When mouse input is received while shift is pressed (regardless of
793  // any other pressed keys), Mac automatically flips the axis. Other
794  // platforms do not do this, so we flip it back to normalize the input
795  // received by the framework. The keyboard+mouse-scroll mechanism is exposed
796  // in the ScrollBehavior of the framework so developers can customize the
797  // behavior.
798  // At time of change, Apple does not expose any other type of API or signal
799  // that the X/Y axes have been flipped.
800  double scaledDeltaX = -event.scrollingDeltaX * pixelsPerLine * scaleFactor;
801  double scaledDeltaY = -event.scrollingDeltaY * pixelsPerLine * scaleFactor;
802  if (event.modifierFlags & NSShiftKeyMask) {
803  flutterEvent.scroll_delta_x = scaledDeltaY;
804  flutterEvent.scroll_delta_y = scaledDeltaX;
805  } else {
806  flutterEvent.scroll_delta_x = scaledDeltaX;
807  flutterEvent.scroll_delta_y = scaledDeltaY;
808  }
809  }
810 
811  [_keyboardManager syncModifiersIfNeeded:event.modifierFlags timestamp:event.timestamp];
812  [_engine sendPointerEvent:flutterEvent];
813 
814  // Update tracking of state as reported to Flutter.
815  if (phase == kDown) {
816  _mouseState.flutter_state_is_down = true;
817  } else if (phase == kUp) {
818  _mouseState.flutter_state_is_down = false;
819  if (_mouseState.has_pending_exit) {
820  [self dispatchMouseEvent:event phase:kRemove];
821  _mouseState.has_pending_exit = false;
822  }
823  } else if (phase == kAdd) {
824  _mouseState.flutter_state_is_added = true;
825  } else if (phase == kRemove) {
826  _mouseState.Reset();
827  }
828 }
829 
830 - (void)onAccessibilityStatusChanged:(BOOL)enabled {
831  if (!enabled && self.viewLoaded && [_textInputPlugin isFirstResponder]) {
832  // Normally TextInputPlugin, when editing, is child of FlutterViewWrapper.
833  // When accessiblity is enabled the TextInputPlugin gets added as an indirect
834  // child to FlutterTextField. When disabling the plugin needs to be reparented
835  // back.
836  [self.view addSubview:_textInputPlugin];
837  }
838 }
839 
840 - (std::shared_ptr<flutter::AccessibilityBridgeMac>)createAccessibilityBridgeWithEngine:
841  (nonnull FlutterEngine*)engine {
842  return std::make_shared<flutter::AccessibilityBridgeMac>(engine, self);
843 }
844 
845 - (nonnull FlutterView*)createFlutterViewWithMTLDevice:(id<MTLDevice>)device
846  commandQueue:(id<MTLCommandQueue>)commandQueue {
847  return [[FlutterView alloc] initWithMTLDevice:device
848  commandQueue:commandQueue
849  delegate:self
850  threadSynchronizer:_threadSynchronizer
851  viewIdentifier:_viewIdentifier];
852 }
853 
854 - (void)onKeyboardLayoutChanged {
855  _keyboardLayoutData = nil;
856  if (_keyboardLayoutNotifier != nil) {
858  }
859 }
860 
861 - (NSString*)lookupKeyForAsset:(NSString*)asset {
862  return [FlutterDartProject lookupKeyForAsset:asset];
863 }
864 
865 - (NSString*)lookupKeyForAsset:(NSString*)asset fromPackage:(NSString*)package {
866  return [FlutterDartProject lookupKeyForAsset:asset fromPackage:package];
867 }
868 
869 #pragma mark - FlutterViewDelegate
870 
871 /**
872  * Responds to view reshape by notifying the engine of the change in dimensions.
873  */
874 - (void)viewDidReshape:(NSView*)view {
875  FML_DCHECK(view == _flutterView);
876  [_engine updateWindowMetricsForViewController:self];
877 }
878 
879 - (BOOL)viewShouldAcceptFirstResponder:(NSView*)view {
880  FML_DCHECK(view == _flutterView);
881  // Only allow FlutterView to become first responder if TextInputPlugin is
882  // not active. Otherwise a mouse event inside FlutterView would cause the
883  // TextInputPlugin to lose first responder status.
884  return !_textInputPlugin.isFirstResponder;
885 }
886 
887 #pragma mark - FlutterPluginRegistry
888 
889 - (id<FlutterPluginRegistrar>)registrarForPlugin:(NSString*)pluginName {
890  return [_engine registrarForPlugin:pluginName];
891 }
892 
893 - (NSObject*)valuePublishedByPlugin:(NSString*)pluginKey {
894  return [_engine valuePublishedByPlugin:pluginKey];
895 }
896 
897 #pragma mark - FlutterKeyboardViewDelegate
898 
899 /**
900  * Returns the current Unicode layout data (kTISPropertyUnicodeKeyLayoutData).
901  *
902  * To use the returned data, convert it to CFDataRef first, finds its bytes
903  * with CFDataGetBytePtr, then reinterpret it into const UCKeyboardLayout*.
904  * It's returned in NSData* to enable auto reference count.
905  */
906 static NSData* CurrentKeyboardLayoutData() {
907  fml::CFRef<TISInputSourceRef> source(TISCopyCurrentKeyboardInputSource());
908  CFTypeRef layout_data = TISGetInputSourceProperty(source, kTISPropertyUnicodeKeyLayoutData);
909  if (layout_data == nil) {
910  // TISGetInputSourceProperty returns null with Japanese keyboard layout.
911  // Using TISCopyCurrentKeyboardLayoutInputSource to fix NULL return.
912  // https://github.com/microsoft/node-native-keymap/blob/5f0699ded00179410a14c0e1b0e089fe4df8e130/src/keyboard_mac.mm#L91
913  source.Reset(TISCopyCurrentKeyboardLayoutInputSource());
914  layout_data = TISGetInputSourceProperty(source, kTISPropertyUnicodeKeyLayoutData);
915  }
916  return (__bridge NSData*)layout_data;
917 }
918 
919 - (void)sendKeyEvent:(const FlutterKeyEvent&)event
920  callback:(nullable FlutterKeyEventCallback)callback
921  userData:(nullable void*)userData {
922  [_engine sendKeyEvent:event callback:callback userData:userData];
923 }
924 
925 - (id<FlutterBinaryMessenger>)getBinaryMessenger {
926  return _engine.binaryMessenger;
927 }
928 
929 - (BOOL)onTextInputKeyEvent:(nonnull NSEvent*)event {
930  return [_textInputPlugin handleKeyEvent:event];
931 }
932 
933 - (void)subscribeToKeyboardLayoutChange:(nullable KeyboardLayoutNotifier)callback {
934  _keyboardLayoutNotifier = callback;
935 }
936 
937 - (LayoutClue)lookUpLayoutForKeyCode:(uint16_t)keyCode shift:(BOOL)shift {
938  if (_keyboardLayoutData == nil) {
939  _keyboardLayoutData = CurrentKeyboardLayoutData();
940  }
941  const UCKeyboardLayout* layout = reinterpret_cast<const UCKeyboardLayout*>(
942  CFDataGetBytePtr((__bridge CFDataRef)_keyboardLayoutData));
943 
944  UInt32 deadKeyState = 0;
945  UniCharCount stringLength = 0;
946  UniChar resultChar;
947 
948  UInt32 modifierState = ((shift ? shiftKey : 0) >> 8) & 0xFF;
949  UInt32 keyboardType = LMGetKbdLast();
950 
951  bool isDeadKey = false;
952  OSStatus status =
953  UCKeyTranslate(layout, keyCode, kUCKeyActionDown, modifierState, keyboardType,
954  kUCKeyTranslateNoDeadKeysBit, &deadKeyState, 1, &stringLength, &resultChar);
955  // For dead keys, press the same key again to get the printable representation of the key.
956  if (status == noErr && stringLength == 0 && deadKeyState != 0) {
957  isDeadKey = true;
958  status =
959  UCKeyTranslate(layout, keyCode, kUCKeyActionDown, modifierState, keyboardType,
960  kUCKeyTranslateNoDeadKeysBit, &deadKeyState, 1, &stringLength, &resultChar);
961  }
962 
963  if (status == noErr && stringLength == 1 && !std::iscntrl(resultChar)) {
964  return LayoutClue{resultChar, isDeadKey};
965  }
966  return LayoutClue{0, false};
967 }
968 
969 - (nonnull NSDictionary*)getPressedState {
970  return [_keyboardManager getPressedState];
971 }
972 
973 #pragma mark - NSResponder
974 
975 - (BOOL)acceptsFirstResponder {
976  return YES;
977 }
978 
979 - (void)keyDown:(NSEvent*)event {
980  [_keyboardManager handleEvent:event];
981 }
982 
983 - (void)keyUp:(NSEvent*)event {
984  [_keyboardManager handleEvent:event];
985 }
986 
987 - (void)flagsChanged:(NSEvent*)event {
988  [_keyboardManager handleEvent:event];
989 }
990 
991 - (void)mouseEntered:(NSEvent*)event {
992  if (_mouseState.has_pending_exit) {
993  _mouseState.has_pending_exit = false;
994  } else {
995  [self dispatchMouseEvent:event phase:kAdd];
996  }
997 }
998 
999 - (void)mouseExited:(NSEvent*)event {
1000  if (_mouseState.buttons != 0) {
1001  _mouseState.has_pending_exit = true;
1002  return;
1003  }
1004  [self dispatchMouseEvent:event phase:kRemove];
1005 }
1006 
1007 - (void)mouseDown:(NSEvent*)event {
1008  _mouseState.buttons |= kFlutterPointerButtonMousePrimary;
1009  [self dispatchMouseEvent:event];
1010 }
1011 
1012 - (void)mouseUp:(NSEvent*)event {
1013  _mouseState.buttons &= ~static_cast<uint64_t>(kFlutterPointerButtonMousePrimary);
1014  [self dispatchMouseEvent:event];
1015 }
1016 
1017 - (void)mouseDragged:(NSEvent*)event {
1018  [self dispatchMouseEvent:event];
1019 }
1020 
1021 - (void)rightMouseDown:(NSEvent*)event {
1022  _mouseState.buttons |= kFlutterPointerButtonMouseSecondary;
1023  [self dispatchMouseEvent:event];
1024 }
1025 
1026 - (void)rightMouseUp:(NSEvent*)event {
1027  _mouseState.buttons &= ~static_cast<uint64_t>(kFlutterPointerButtonMouseSecondary);
1028  [self dispatchMouseEvent:event];
1029 }
1030 
1031 - (void)rightMouseDragged:(NSEvent*)event {
1032  [self dispatchMouseEvent:event];
1033 }
1034 
1035 - (void)otherMouseDown:(NSEvent*)event {
1036  _mouseState.buttons |= (1 << event.buttonNumber);
1037  [self dispatchMouseEvent:event];
1038 }
1039 
1040 - (void)otherMouseUp:(NSEvent*)event {
1041  _mouseState.buttons &= ~static_cast<uint64_t>(1 << event.buttonNumber);
1042  [self dispatchMouseEvent:event];
1043 }
1044 
1045 - (void)otherMouseDragged:(NSEvent*)event {
1046  [self dispatchMouseEvent:event];
1047 }
1048 
1049 - (void)mouseMoved:(NSEvent*)event {
1050  [self dispatchMouseEvent:event];
1051 }
1052 
1053 - (void)scrollWheel:(NSEvent*)event {
1054  [self dispatchGestureEvent:event];
1055 }
1056 
1057 - (void)magnifyWithEvent:(NSEvent*)event {
1058  [self dispatchGestureEvent:event];
1059 }
1060 
1061 - (void)rotateWithEvent:(NSEvent*)event {
1062  [self dispatchGestureEvent:event];
1063 }
1064 
1065 - (void)swipeWithEvent:(NSEvent*)event {
1066  // Not needed, it's handled by scrollWheel.
1067 }
1068 
1069 - (void)touchesBeganWithEvent:(NSEvent*)event {
1070  NSTouch* touch = event.allTouches.anyObject;
1071  if (touch != nil) {
1072  if ((event.timestamp - _mouseState.last_scroll_momentum_changed_time) <
1073  kTrackpadTouchInertiaCancelWindowMs) {
1074  // The trackpad has been touched following a scroll momentum event.
1075  // A scroll inertia cancel message should be sent to the framework.
1076  NSPoint locationInView = [self.flutterView convertPoint:event.locationInWindow fromView:nil];
1077  NSPoint locationInBackingCoordinates =
1078  [self.flutterView convertPointToBacking:locationInView];
1079  FlutterPointerEvent flutterEvent = {
1080  .struct_size = sizeof(flutterEvent),
1081  .timestamp = static_cast<size_t>(event.timestamp * USEC_PER_SEC),
1082  .x = locationInBackingCoordinates.x,
1083  .y = -locationInBackingCoordinates.y, // convertPointToBacking makes this negative.
1084  .device = kPointerPanZoomDeviceId,
1085  .signal_kind = kFlutterPointerSignalKindScrollInertiaCancel,
1086  .device_kind = kFlutterPointerDeviceKindTrackpad,
1087  .view_id = static_cast<FlutterViewIdentifier>(_viewIdentifier),
1088  };
1089 
1090  [_engine sendPointerEvent:flutterEvent];
1091  // Ensure no further scroll inertia cancel event will be sent.
1092  _mouseState.last_scroll_momentum_changed_time = 0;
1093  }
1094  }
1095 }
1096 
1097 @end
flutter::LayoutClue
Definition: FlutterKeyboardViewDelegate.h:20
FlutterEngine
Definition: FlutterEngine.h:31
FlutterViewController
Definition: FlutterViewController.h:73
FlutterEngine.h
FlutterViewWrapper
Definition: FlutterViewController.mm:154
FlutterEngine_Internal.h
+[FlutterDartProject lookupKeyForAsset:]
NSString * lookupKeyForAsset:(NSString *asset)
Definition: FlutterDartProject.mm:116
_keyboardLayoutNotifier
flutter::KeyboardLayoutNotifier _keyboardLayoutNotifier
Definition: FlutterKeyboardManagerTest.mm:243
_bridge
std::shared_ptr< flutter::AccessibilityBridgeMac > _bridge
Definition: FlutterViewController.mm:333
FlutterChannels.h
FlutterRenderer.h
_project
FlutterDartProject * _project
Definition: FlutterEngine.mm:409
FlutterViewController::engine
FlutterEngine * engine
Definition: FlutterViewController.h:78
FlutterPluginRegistrar-p
Definition: FlutterPluginRegistrarMacOS.h:28
FlutterKeyPrimaryResponder.h
-[FlutterView setBackgroundColor:]
void setBackgroundColor:(nonnull NSColor *color)
OnKeyboardLayoutChanged
static void OnKeyboardLayoutChanged(CFNotificationCenterRef center, void *observer, CFStringRef name, const void *object, CFDictionaryRef userInfo)
Definition: FlutterViewController.mm:239
_controller
__weak FlutterViewController * _controller
Definition: FlutterViewController.mm:250
FlutterThreadSynchronizer
Definition: FlutterThreadSynchronizer.h:18
flutter::KeyboardLayoutNotifier
void(^ KeyboardLayoutNotifier)()
Definition: FlutterKeyboardViewDelegate.h:16
FlutterTextInputPlugin
Definition: FlutterTextInputPlugin.h:27
FlutterCodecs.h
FlutterKeyboardManager.h
FlutterViewController_Internal.h
_threadSynchronizer
FlutterThreadSynchronizer * _threadSynchronizer
Definition: FlutterViewController.mm:341
FlutterViewController::viewIdentifier
FlutterViewIdentifier viewIdentifier
Definition: FlutterViewController.h:130
FlutterView
Definition: FlutterView.h:35
FlutterTextInputSemanticsObject.h
+[FlutterDartProject lookupKeyForAsset:fromPackage:]
NSString * lookupKeyForAsset:fromPackage:(NSString *asset,[fromPackage] NSString *package)
Definition: FlutterDartProject.mm:125
FlutterDartProject
Definition: FlutterDartProject.mm:24
FlutterKeyboardManager
Definition: FlutterKeyboardManager.h:27
FlutterBinaryMessenger-p
Definition: FlutterBinaryMessenger.h:49
FlutterView.h
FlutterViewIdentifier
int64_t FlutterViewIdentifier
Definition: FlutterViewController.h:21
FlutterViewDelegate-p
Definition: FlutterView.h:18
FlutterViewController.h