Flutter macOS Embedder
AccessibilityBridgeMac.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
6 
11 #include "flutter/shell/platform/embedder/embedder.h"
12 
13 namespace flutter {
14 
15 // Native mac notifications fired. These notifications are not publicly documented.
16 static NSString* const kAccessibilityLoadCompleteNotification = @"AXLoadComplete";
17 static NSString* const kAccessibilityInvalidStatusChangedNotification = @"AXInvalidStatusChanged";
18 static NSString* const kAccessibilityLiveRegionCreatedNotification = @"AXLiveRegionCreated";
19 static NSString* const kAccessibilityLiveRegionChangedNotification = @"AXLiveRegionChanged";
20 static NSString* const kAccessibilityExpandedChanged = @"AXExpandedChanged";
21 static NSString* const kAccessibilityMenuItemSelectedNotification = @"AXMenuItemSelected";
22 
24  __weak FlutterViewController* view_controller)
25  : flutter_engine_(flutter_engine), view_controller_(view_controller) {}
26 
28  ui::AXEventGenerator::TargetedEvent targeted_event) {
29  if (!view_controller_.viewLoaded || !view_controller_.view.window) {
30  // Don't need to send accessibility events if the there is no view or window.
31  return;
32  }
33  ui::AXNode* ax_node = targeted_event.node;
34  std::vector<AccessibilityBridgeMac::NSAccessibilityEvent> events =
35  MacOSEventsFromAXEvent(targeted_event.event_params.event, *ax_node);
36  for (const AccessibilityBridgeMac::NSAccessibilityEvent& event : events) {
37  if (event.user_info != nil) {
38  DispatchMacOSNotificationWithUserInfo(event.target, event.name, event.user_info);
39  } else {
40  DispatchMacOSNotification(event.target, event.name);
41  }
42  }
43 }
44 
45 std::vector<AccessibilityBridgeMac::NSAccessibilityEvent>
46 AccessibilityBridgeMac::MacOSEventsFromAXEvent(ui::AXEventGenerator::Event event_type,
47  const ui::AXNode& ax_node) const {
48  // Gets the native_node with the node_id.
49  NSCAssert(flutter_engine_, @"Flutter engine should not be deallocated");
50  auto platform_node_delegate = GetFlutterPlatformNodeDelegateFromID(ax_node.id()).lock();
51  NSCAssert(platform_node_delegate, @"Event target must exist in accessibility bridge.");
52  auto mac_platform_node_delegate =
53  std::static_pointer_cast<FlutterPlatformNodeDelegateMac>(platform_node_delegate);
54  gfx::NativeViewAccessible native_node = mac_platform_node_delegate->GetNativeViewAccessible();
55 
56  std::vector<AccessibilityBridgeMac::NSAccessibilityEvent> events;
57  switch (event_type) {
58  case ui::AXEventGenerator::Event::ACTIVE_DESCENDANT_CHANGED:
59  if (ax_node.data().role == ax::mojom::Role::kTree) {
60  events.push_back({
61  .name = NSAccessibilitySelectedRowsChangedNotification,
62  .target = native_node,
63  .user_info = nil,
64  });
65  } else if (ax_node.data().role == ax::mojom::Role::kTextFieldWithComboBox) {
66  // Even though the selected item in the combo box has changed, don't
67  // post a focus change because this will take the focus out of
68  // the combo box where the user might be typing.
69  events.push_back({
70  .name = NSAccessibilitySelectedChildrenChangedNotification,
71  .target = native_node,
72  .user_info = nil,
73  });
74  }
75  // In all other cases, this delegate should post
76  // |NSAccessibilityFocusedUIElementChangedNotification|, but this is
77  // handled elsewhere.
78  break;
79  case ui::AXEventGenerator::Event::LOAD_COMPLETE:
80  events.push_back({
82  .target = native_node,
83  .user_info = nil,
84  });
85  break;
86  case ui::AXEventGenerator::Event::INVALID_STATUS_CHANGED:
87  events.push_back({
89  .target = native_node,
90  .user_info = nil,
91  });
92  break;
93  case ui::AXEventGenerator::Event::SELECTED_CHILDREN_CHANGED:
94  if (ui::IsTableLike(ax_node.data().role)) {
95  events.push_back({
96  .name = NSAccessibilitySelectedRowsChangedNotification,
97  .target = native_node,
98  .user_info = nil,
99  });
100  } else {
101  // VoiceOver does not read anything if selection changes on the
102  // currently focused object, and the focus did not move. Fire a
103  // selection change if the focus did not change.
104  NSAccessibilityElement* native_accessibility_node = (NSAccessibilityElement*)native_node;
105  if (native_accessibility_node.accessibilityFocusedUIElement &&
106  ax_node.data().HasState(ax::mojom::State::kMultiselectable) &&
107  !HasPendingEvent(ui::AXEventGenerator::Event::ACTIVE_DESCENDANT_CHANGED) &&
108  !HasPendingEvent(ui::AXEventGenerator::Event::FOCUS_CHANGED)) {
109  // Don't fire selected children change, it will sometimes override
110  // announcement of current focus.
111  break;
112  }
113  events.push_back({
114  .name = NSAccessibilitySelectedChildrenChangedNotification,
115  .target = native_node,
116  .user_info = nil,
117  });
118  }
119  break;
120  case ui::AXEventGenerator::Event::DOCUMENT_SELECTION_CHANGED: {
121  id focused = mac_platform_node_delegate->GetFocus();
122  if ([focused isKindOfClass:[FlutterTextField class]]) {
123  // If it is a text field, the selection notifications are handled by
124  // the FlutterTextField directly. Only need to make sure it is the
125  // first responder.
126  FlutterTextField* native_text_field = (FlutterTextField*)focused;
127  if (native_text_field == mac_platform_node_delegate->GetFocus()) {
128  [native_text_field startEditing];
129  }
130  break;
131  }
132  // This event always fires at root
133  events.push_back({
134  .name = NSAccessibilitySelectedTextChangedNotification,
135  .target = native_node,
136  .user_info = nil,
137  });
138  // WebKit fires a notification both on the focused object and the page
139  // root.
140  const ui::AXTreeData& tree_data = GetAXTreeData();
141  int32_t focus = tree_data.focus_id;
142  if (focus == ui::AXNode::kInvalidAXID || focus != tree_data.sel_anchor_object_id) {
143  break; // Just fire a notification on the root.
144  }
145  auto focus_node = GetFlutterPlatformNodeDelegateFromID(focus).lock();
146  if (!focus_node) {
147  break; // Just fire a notification on the root.
148  }
149  events.push_back({
150  .name = NSAccessibilitySelectedTextChangedNotification,
151  .target = focus_node->GetNativeViewAccessible(),
152  .user_info = nil,
153  });
154  break;
155  }
156  case ui::AXEventGenerator::Event::CHECKED_STATE_CHANGED:
157  events.push_back({
158  .name = NSAccessibilityValueChangedNotification,
159  .target = native_node,
160  .user_info = nil,
161  });
162  break;
163  case ui::AXEventGenerator::Event::VALUE_CHANGED: {
164  if (ax_node.data().role == ax::mojom::Role::kTextField) {
165  // If it is a text field, the value change notifications are handled by
166  // the FlutterTextField directly. Only need to make sure it is the
167  // first responder.
168  FlutterTextField* native_text_field =
169  (FlutterTextField*)mac_platform_node_delegate->GetNativeViewAccessible();
170  id focused = mac_platform_node_delegate->GetFocus();
171  if (!focused || native_text_field == focused) {
172  [native_text_field startEditing];
173  }
174  break;
175  }
176  events.push_back({
177  .name = NSAccessibilityValueChangedNotification,
178  .target = native_node,
179  .user_info = nil,
180  });
181  if (ax_node.data().HasState(ax::mojom::State::kEditable)) {
182  events.push_back({
183  .name = NSAccessibilityValueChangedNotification,
184  .target = RootDelegate()->GetNativeViewAccessible(),
185  .user_info = nil,
186  });
187  }
188  break;
189  }
190  case ui::AXEventGenerator::Event::LIVE_REGION_CREATED:
191  events.push_back({
193  .target = native_node,
194  .user_info = nil,
195  });
196  break;
197  case ui::AXEventGenerator::Event::ALERT: {
198  events.push_back({
200  .target = native_node,
201  .user_info = nil,
202  });
203  // VoiceOver requires a live region changed notification to actually
204  // announce the live region.
205  auto live_region_events =
206  MacOSEventsFromAXEvent(ui::AXEventGenerator::Event::LIVE_REGION_CHANGED, ax_node);
207  events.insert(events.end(), live_region_events.begin(), live_region_events.end());
208  break;
209  }
210  case ui::AXEventGenerator::Event::LIVE_REGION_CHANGED: {
211  // Uses native VoiceOver support for live regions.
212  events.push_back({
214  .target = native_node,
215  .user_info = nil,
216  });
217  break;
218  }
219  case ui::AXEventGenerator::Event::ROW_COUNT_CHANGED:
220  events.push_back({
221  .name = NSAccessibilityRowCountChangedNotification,
222  .target = native_node,
223  .user_info = nil,
224  });
225  break;
226  case ui::AXEventGenerator::Event::EXPANDED: {
227  NSAccessibilityNotificationName mac_notification;
228  if (ax_node.data().role == ax::mojom::Role::kRow ||
229  ax_node.data().role == ax::mojom::Role::kTreeItem) {
230  mac_notification = NSAccessibilityRowExpandedNotification;
231  } else {
232  mac_notification = kAccessibilityExpandedChanged;
233  }
234  events.push_back({
235  .name = mac_notification,
236  .target = native_node,
237  .user_info = nil,
238  });
239  break;
240  }
241  case ui::AXEventGenerator::Event::COLLAPSED: {
242  NSAccessibilityNotificationName mac_notification;
243  if (ax_node.data().role == ax::mojom::Role::kRow ||
244  ax_node.data().role == ax::mojom::Role::kTreeItem) {
245  mac_notification = NSAccessibilityRowCollapsedNotification;
246  } else {
247  mac_notification = kAccessibilityExpandedChanged;
248  }
249  events.push_back({
250  .name = mac_notification,
251  .target = native_node,
252  .user_info = nil,
253  });
254  break;
255  }
256  case ui::AXEventGenerator::Event::MENU_ITEM_SELECTED:
257  events.push_back({
259  .target = native_node,
260  .user_info = nil,
261  });
262  break;
263  case ui::AXEventGenerator::Event::CHILDREN_CHANGED: {
264  // NSAccessibilityCreatedNotification seems to be the only way to let
265  // Voiceover pick up layout changes.
266  events.push_back({
267  .name = NSAccessibilityCreatedNotification,
268  .target = view_controller_.view.window,
269  .user_info = nil,
270  });
271  break;
272  }
273  case ui::AXEventGenerator::Event::SUBTREE_CREATED:
274  case ui::AXEventGenerator::Event::ACCESS_KEY_CHANGED:
275  case ui::AXEventGenerator::Event::ATK_TEXT_OBJECT_ATTRIBUTE_CHANGED:
276  case ui::AXEventGenerator::Event::ATOMIC_CHANGED:
277  case ui::AXEventGenerator::Event::AUTO_COMPLETE_CHANGED:
278  case ui::AXEventGenerator::Event::BUSY_CHANGED:
279  case ui::AXEventGenerator::Event::CONTROLS_CHANGED:
280  case ui::AXEventGenerator::Event::CLASS_NAME_CHANGED:
281  case ui::AXEventGenerator::Event::DESCRIBED_BY_CHANGED:
282  case ui::AXEventGenerator::Event::DESCRIPTION_CHANGED:
283  case ui::AXEventGenerator::Event::DOCUMENT_TITLE_CHANGED:
284  case ui::AXEventGenerator::Event::DROPEFFECT_CHANGED:
285  case ui::AXEventGenerator::Event::ENABLED_CHANGED:
286  case ui::AXEventGenerator::Event::FOCUS_CHANGED:
287  case ui::AXEventGenerator::Event::FLOW_FROM_CHANGED:
288  case ui::AXEventGenerator::Event::FLOW_TO_CHANGED:
289  case ui::AXEventGenerator::Event::GRABBED_CHANGED:
290  case ui::AXEventGenerator::Event::HASPOPUP_CHANGED:
291  case ui::AXEventGenerator::Event::HIERARCHICAL_LEVEL_CHANGED:
292  case ui::AXEventGenerator::Event::IGNORED_CHANGED:
293  case ui::AXEventGenerator::Event::IMAGE_ANNOTATION_CHANGED:
294  case ui::AXEventGenerator::Event::KEY_SHORTCUTS_CHANGED:
295  case ui::AXEventGenerator::Event::LABELED_BY_CHANGED:
296  case ui::AXEventGenerator::Event::LANGUAGE_CHANGED:
297  case ui::AXEventGenerator::Event::LAYOUT_INVALIDATED:
298  case ui::AXEventGenerator::Event::LIVE_REGION_NODE_CHANGED:
299  case ui::AXEventGenerator::Event::LIVE_RELEVANT_CHANGED:
300  case ui::AXEventGenerator::Event::LIVE_STATUS_CHANGED:
301  case ui::AXEventGenerator::Event::LOAD_START:
302  case ui::AXEventGenerator::Event::MULTILINE_STATE_CHANGED:
303  case ui::AXEventGenerator::Event::MULTISELECTABLE_STATE_CHANGED:
304  case ui::AXEventGenerator::Event::NAME_CHANGED:
305  case ui::AXEventGenerator::Event::OBJECT_ATTRIBUTE_CHANGED:
306  case ui::AXEventGenerator::Event::OTHER_ATTRIBUTE_CHANGED:
307  case ui::AXEventGenerator::Event::PLACEHOLDER_CHANGED:
308  case ui::AXEventGenerator::Event::PORTAL_ACTIVATED:
309  case ui::AXEventGenerator::Event::POSITION_IN_SET_CHANGED:
310  case ui::AXEventGenerator::Event::READONLY_CHANGED:
311  case ui::AXEventGenerator::Event::RELATED_NODE_CHANGED:
312  case ui::AXEventGenerator::Event::REQUIRED_STATE_CHANGED:
313  case ui::AXEventGenerator::Event::ROLE_CHANGED:
314  case ui::AXEventGenerator::Event::SCROLL_HORIZONTAL_POSITION_CHANGED:
315  case ui::AXEventGenerator::Event::SCROLL_VERTICAL_POSITION_CHANGED:
316  case ui::AXEventGenerator::Event::SELECTED_CHANGED:
317  case ui::AXEventGenerator::Event::SET_SIZE_CHANGED:
318  case ui::AXEventGenerator::Event::SORT_CHANGED:
319  case ui::AXEventGenerator::Event::STATE_CHANGED:
320  case ui::AXEventGenerator::Event::TEXT_ATTRIBUTE_CHANGED:
321  case ui::AXEventGenerator::Event::VALUE_MAX_CHANGED:
322  case ui::AXEventGenerator::Event::VALUE_MIN_CHANGED:
323  case ui::AXEventGenerator::Event::VALUE_STEP_CHANGED:
324  case ui::AXEventGenerator::Event::WIN_IACCESSIBLE_STATE_CHANGED:
325  // There are some notifications that aren't meaningful on Mac.
326  // It's okay to skip them.
327  break;
328  }
329  return events;
330 }
331 
333  FlutterSemanticsAction action,
334  fml::MallocMapping data) {
335  NSCAssert(flutter_engine_, @"Flutter engine should not be deallocated");
336  NSCAssert(view_controller_.viewLoaded && view_controller_.view.window,
337  @"The accessibility bridge should not receive accessibility actions if the flutter view"
338  @"is not loaded or attached to a NSWindow.");
339  [flutter_engine_ dispatchSemanticsAction:action toTarget:target withData:std::move(data)];
340 }
341 
342 std::shared_ptr<FlutterPlatformNodeDelegate>
344  return std::make_shared<FlutterPlatformNodeDelegateMac>(weak_from_this(), view_controller_);
345 }
346 
347 // Private method
348 void AccessibilityBridgeMac::DispatchMacOSNotification(
349  gfx::NativeViewAccessible native_node,
350  NSAccessibilityNotificationName mac_notification) {
351  NSCAssert(mac_notification, @"The notification must not be null.");
352  NSCAssert(native_node, @"The notification target must not be null.");
353  NSAccessibilityPostNotification(native_node, mac_notification);
354 }
355 
356 void AccessibilityBridgeMac::DispatchMacOSNotificationWithUserInfo(
357  gfx::NativeViewAccessible native_node,
358  NSAccessibilityNotificationName mac_notification,
359  NSDictionary* user_info) {
360  NSCAssert(mac_notification, @"The notification must not be null.");
361  NSCAssert(native_node, @"The notification target must not be null.");
362  NSCAssert(user_info, @"The notification data must not be null.");
363  NSAccessibilityPostNotificationWithUserInfo(native_node, mac_notification, user_info);
364 }
365 
366 bool AccessibilityBridgeMac::HasPendingEvent(ui::AXEventGenerator::Event event) const {
367  NSCAssert(flutter_engine_, @"Flutter engine should not be deallocated");
368  std::vector<ui::AXEventGenerator::TargetedEvent> pending_events = GetPendingEvents();
369  for (const auto& pending_event : GetPendingEvents()) {
370  if (pending_event.event_params.event == event) {
371  return true;
372  }
373  }
374  return false;
375 }
376 
377 } // namespace flutter
flutter::kAccessibilityInvalidStatusChangedNotification
static NSString *const kAccessibilityInvalidStatusChangedNotification
Definition: AccessibilityBridgeMac.mm:17
flutter::kAccessibilityLiveRegionCreatedNotification
static NSString *const kAccessibilityLiveRegionCreatedNotification
Definition: AccessibilityBridgeMac.mm:18
FlutterEngine
Definition: FlutterEngine.h:30
FlutterViewController
Definition: FlutterViewController.h:73
flutter::AccessibilityBridgeMac::CreateFlutterPlatformNodeDelegate
std::shared_ptr< FlutterPlatformNodeDelegate > CreateFlutterPlatformNodeDelegate() override
Creates a platform specific FlutterPlatformNodeDelegate. Ownership passes to the caller....
Definition: AccessibilityBridgeMac.mm:343
flutter::AccessibilityBridgeMac::DispatchAccessibilityAction
void DispatchAccessibilityAction(AccessibilityNodeId target, FlutterSemanticsAction action, fml::MallocMapping data) override
Dispatch accessibility action back to the Flutter framework. These actions are generated in the nativ...
Definition: AccessibilityBridgeMac.mm:332
FlutterEngine_Internal.h
AccessibilityBridgeMac.h
flutter::kAccessibilityLoadCompleteNotification
static NSString *const kAccessibilityLoadCompleteNotification
Definition: AccessibilityBridgeMac.mm:16
flutter::kAccessibilityLiveRegionChangedNotification
static NSString *const kAccessibilityLiveRegionChangedNotification
Definition: AccessibilityBridgeMac.mm:19
flutter::AccessibilityBridgeMac::OnAccessibilityEvent
void OnAccessibilityEvent(ui::AXEventGenerator::TargetedEvent targeted_event) override
Handle accessibility events generated due to accessibility tree changes. These events are needed to b...
Definition: AccessibilityBridgeMac.mm:27
FlutterPlatformNodeDelegateMac.h
flutter::AccessibilityBridge::GetFlutterPlatformNodeDelegateFromID
std::weak_ptr< FlutterPlatformNodeDelegate > GetFlutterPlatformNodeDelegateFromID(AccessibilityNodeId id) const
Get the flutter platform node delegate with the given id from this accessibility bridge....
Definition: accessibility_bridge.cc:131
flutter
Definition: AccessibilityBridgeMac.h:16
FlutterViewController_Internal.h
flutter::AccessibilityBridge::GetAXTreeData
const ui::AXTreeData & GetAXTreeData() const
Get the ax tree data from this accessibility bridge. The tree data contains information such as the i...
Definition: accessibility_bridge.cc:141
flutter::kAccessibilityExpandedChanged
static NSString *const kAccessibilityExpandedChanged
Definition: AccessibilityBridgeMac.mm:20
flutter::AccessibilityBridgeMac::AccessibilityBridgeMac
AccessibilityBridgeMac(__weak FlutterEngine *flutter_engine, __weak FlutterViewController *view_controller)
Creates an AccessibilityBridgeMacDelegate.
Definition: AccessibilityBridgeMac.mm:23
FlutterTextInputSemanticsObject.h
flutter::AccessibilityBridge::GetPendingEvents
const std::vector< ui::AXEventGenerator::TargetedEvent > GetPendingEvents() const
Gets all pending accessibility events generated during semantics updates. This is useful when decidin...
Definition: accessibility_bridge.cc:146
flutter::AccessibilityBridge::RootDelegate
ui::AXPlatformNodeDelegate * RootDelegate() const override
Definition: accessibility_bridge.cc:721
flutter::kAccessibilityMenuItemSelectedNotification
static NSString *const kAccessibilityMenuItemSelectedNotification
Definition: AccessibilityBridgeMac.mm:21
FlutterTextField
Definition: FlutterTextInputSemanticsObject.h:81