10 #include "flutter/third_party/accessibility/ax/ax_tree_manager_map.h"
11 #include "flutter/third_party/accessibility/ax/ax_tree_update.h"
12 #include "flutter/third_party/accessibility/base/logging.h"
17 FlutterSemanticsAction::kFlutterSemanticsActionScrollLeft |
18 FlutterSemanticsAction::kFlutterSemanticsActionScrollRight |
19 FlutterSemanticsAction::kFlutterSemanticsActionScrollUp |
20 FlutterSemanticsAction::kFlutterSemanticsActionScrollDown;
24 : tree_(std::make_unique<ui::AXTree>()) {
25 event_generator_.SetTree(tree_.get());
26 tree_->AddObserver(
static_cast<ui::AXTreeObserver*
>(
this));
27 ui::AXTreeData data = tree_->data();
28 data.tree_id = ui::AXTreeID::CreateNewAXTreeID();
29 tree_->UpdateData(data);
30 ui::AXTreeManagerMap::GetInstance().AddTreeManager(tree_->GetAXTreeID(),
35 event_generator_.ReleaseTree();
36 tree_->RemoveObserver(
static_cast<ui::AXTreeObserver*
>(
this));
40 const FlutterSemanticsNode2& node) {
41 pending_semantics_node_updates_[node.id] = FromFlutterSemanticsNode(node);
45 const FlutterSemanticsCustomAction2& action) {
46 pending_semantics_custom_action_updates_[action.id] =
47 FromFlutterSemanticsCustomAction(action);
58 std::optional<ui::AXTreeUpdate> remove_reparented =
59 CreateRemoveReparentedNodesUpdate();
60 if (remove_reparented.has_value()) {
61 tree_->Unserialize(remove_reparented.value());
63 std::string
error = tree_->error();
65 FML_LOG(ERROR) <<
"Failed to update ui::AXTree, error: " <<
error;
73 ui::AXTreeUpdate update{.tree_data = tree_->data()};
83 std::vector<std::vector<SemanticsNode>> results;
84 while (!pending_semantics_node_updates_.empty()) {
85 auto begin = pending_semantics_node_updates_.begin();
86 SemanticsNode
target = begin->second;
87 std::vector<SemanticsNode> sub_tree_list;
88 GetSubTreeList(
target, sub_tree_list);
89 results.push_back(sub_tree_list);
90 pending_semantics_node_updates_.erase(begin);
93 for (
size_t i = results.size();
i > 0;
i--) {
94 for (
const SemanticsNode& node : results[
i - 1]) {
95 ConvertFlutterUpdate(node, update);
102 if (!results.empty() &&
GetRootAsAXNode()->
id() == ui::AXNode::kInvalidAXID) {
103 FML_DCHECK(!results.back().empty());
105 update.root_id = results.back().front().id;
108 tree_->Unserialize(update);
109 pending_semantics_node_updates_.clear();
110 pending_semantics_custom_action_updates_.clear();
112 std::string
error = tree_->error();
113 if (!
error.empty()) {
114 FML_LOG(ERROR) <<
"Failed to update ui::AXTree, error: " <<
error;
118 for (
const auto& targeted_event : event_generator_) {
121 if (event_target.expired()) {
127 event_generator_.ClearEvents();
130 std::weak_ptr<FlutterPlatformNodeDelegate>
133 const auto iter = id_wrapper_map_.find(
id);
134 if (iter != id_wrapper_map_.end()) {
138 return std::weak_ptr<FlutterPlatformNodeDelegate>();
142 return tree_->data();
145 const std::vector<ui::AXEventGenerator::TargetedEvent>
147 std::vector<ui::AXEventGenerator::TargetedEvent>
result(
148 event_generator_.begin(), event_generator_.end());
152 void AccessibilityBridge::OnNodeWillBeDeleted(ui::AXTree* tree,
155 void AccessibilityBridge::OnSubtreeWillBeDeleted(ui::AXTree* tree,
158 void AccessibilityBridge::OnNodeReparented(ui::AXTree* tree, ui::AXNode* node) {
161 void AccessibilityBridge::OnRoleChanged(ui::AXTree* tree,
163 ax::mojom::Role old_role,
164 ax::mojom::Role new_role) {}
166 void AccessibilityBridge::OnNodeCreated(ui::AXTree* tree, ui::AXNode* node) {
169 id_wrapper_map_[node->id()]->Init(
170 std::static_pointer_cast<FlutterPlatformNodeDelegate::OwnerBridge>(
175 void AccessibilityBridge::OnNodeDeleted(ui::AXTree* tree,
177 BASE_DCHECK(node_id != ui::AXNode::kInvalidAXID);
178 if (id_wrapper_map_.find(node_id) != id_wrapper_map_.end()) {
179 id_wrapper_map_.erase(node_id);
183 void AccessibilityBridge::OnAtomicUpdateFinished(
186 const std::vector<ui::AXTreeObserver::Change>& changes) {
190 for (
const auto& change : changes) {
191 ui::AXNode* node = change.node;
192 const ui::AXNodeData& data = node->data();
194 if (node->parent()) {
195 offset_container_id = node->parent()->id();
197 node->SetLocation(offset_container_id, data.relative_bounds.bounds,
198 data.relative_bounds.transform.get());
202 std::optional<ui::AXTreeUpdate>
203 AccessibilityBridge::CreateRemoveReparentedNodesUpdate() {
204 std::unordered_map<int32_t, ui::AXNodeData> updates;
206 for (
const auto& node_update : pending_semantics_node_updates_) {
207 for (int32_t child_id : node_update.second.children_in_traversal_order) {
209 ui::AXNode* child = tree_->GetFromId(child_id);
215 assert(child->parent());
218 if (child->parent()->id() == node_update.second.id) {
224 assert(pending_semantics_node_updates_.find(child_id) !=
225 pending_semantics_node_updates_.end());
228 int32_t parent_id = child->parent()->id();
229 if (updates.find(parent_id) == updates.end()) {
230 updates[parent_id] = tree_->GetFromId(parent_id)->data();
233 ui::AXNodeData* parent = &updates[parent_id];
234 auto iter = std::find(parent->child_ids.begin(), parent->child_ids.end(),
237 assert(iter != parent->child_ids.end());
238 parent->child_ids.erase(iter);
242 if (updates.empty()) {
246 ui::AXTreeUpdate update{
247 .tree_data = tree_->data(),
248 .nodes = std::vector<ui::AXNodeData>(),
251 for (std::pair<int32_t, ui::AXNodeData> data : updates) {
252 update.nodes.push_back(std::move(data.second));
259 void AccessibilityBridge::GetSubTreeList(
const SemanticsNode&
target,
260 std::vector<SemanticsNode>&
result) {
262 for (int32_t child :
target.children_in_traversal_order) {
263 auto iter = pending_semantics_node_updates_.find(child);
264 if (iter != pending_semantics_node_updates_.end()) {
265 SemanticsNode node = iter->second;
266 GetSubTreeList(node,
result);
267 pending_semantics_node_updates_.erase(iter);
272 void AccessibilityBridge::ConvertFlutterUpdate(
const SemanticsNode& node,
273 ui::AXTreeUpdate& tree_update) {
274 ui::AXNodeData node_data;
275 node_data.id = node.id;
276 SetRoleFromFlutterUpdate(node_data, node);
277 SetStateFromFlutterUpdate(node_data, node);
278 SetActionsFromFlutterUpdate(node_data, node);
279 SetBooleanAttributesFromFlutterUpdate(node_data, node);
280 SetIntAttributesFromFlutterUpdate(node_data, node);
281 SetIntListAttributesFromFlutterUpdate(node_data, node);
282 SetStringListAttributesFromFlutterUpdate(node_data, node);
283 SetNameFromFlutterUpdate(node_data, node);
284 SetValueFromFlutterUpdate(node_data, node);
285 SetTooltipFromFlutterUpdate(node_data, node);
286 node_data.relative_bounds.bounds.SetRect(node.rect.left, node.rect.top,
287 node.rect.right - node.rect.left,
288 node.rect.bottom - node.rect.top);
289 node_data.relative_bounds.transform = std::make_unique<gfx::Transform>(
290 node.transform.scaleX, node.transform.skewX, node.transform.transX, 0,
291 node.transform.skewY, node.transform.scaleY, node.transform.transY, 0,
292 node.transform.pers0, node.transform.pers1, node.transform.pers2, 0, 0, 0,
294 for (
auto child : node.children_in_traversal_order) {
295 node_data.child_ids.push_back(child);
297 SetTreeData(node, tree_update);
298 tree_update.nodes.push_back(node_data);
301 void AccessibilityBridge::SetRoleFromFlutterUpdate(ui::AXNodeData& node_data,
302 const SemanticsNode& node) {
303 FlutterSemanticsFlag
flags = node.flags;
304 if (
flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsButton) {
305 node_data.role = ax::mojom::Role::kButton;
308 if (
flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField &&
309 !(
flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsReadOnly)) {
310 node_data.role = ax::mojom::Role::kTextField;
313 if (
flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsHeader) {
314 node_data.role = ax::mojom::Role::kHeader;
317 if (
flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsImage) {
318 node_data.role = ax::mojom::Role::kImage;
321 if (
flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsLink) {
322 node_data.role = ax::mojom::Role::kLink;
326 if (
flags & kFlutterSemanticsFlagIsInMutuallyExclusiveGroup &&
327 flags & kFlutterSemanticsFlagHasCheckedState) {
328 node_data.role = ax::mojom::Role::kRadioButton;
331 if (
flags & kFlutterSemanticsFlagHasCheckedState) {
332 node_data.role = ax::mojom::Role::kCheckBox;
335 if (
flags & kFlutterSemanticsFlagHasToggledState) {
336 node_data.role = ax::mojom::Role::kSwitch;
339 if (
flags & kFlutterSemanticsFlagIsSlider) {
340 node_data.role = ax::mojom::Role::kSlider;
345 if (node.children_in_traversal_order.empty()) {
346 node_data.role = ax::mojom::Role::kStaticText;
348 node_data.role = ax::mojom::Role::kGroup;
352 void AccessibilityBridge::SetStateFromFlutterUpdate(ui::AXNodeData& node_data,
353 const SemanticsNode& node) {
354 FlutterSemanticsFlag
flags = node.flags;
355 FlutterSemanticsAction actions = node.actions;
356 if (
flags & FlutterSemanticsFlag::kFlutterSemanticsFlagHasExpandedState &&
357 flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsExpanded) {
358 node_data.AddState(ax::mojom::State::kExpanded);
360 FlutterSemanticsFlag::kFlutterSemanticsFlagHasExpandedState) {
361 node_data.AddState(ax::mojom::State::kCollapsed);
363 if (
flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField &&
364 (
flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsReadOnly) == 0) {
365 node_data.AddState(ax::mojom::State::kEditable);
367 if (node_data.role == ax::mojom::Role::kStaticText &&
369 node.label.empty() && node.hint.empty()) {
370 node_data.AddState(ax::mojom::State::kIgnored);
375 node_data.AddState(ax::mojom::State::kFocusable);
379 void AccessibilityBridge::SetActionsFromFlutterUpdate(
380 ui::AXNodeData& node_data,
381 const SemanticsNode& node) {
382 FlutterSemanticsAction actions = node.actions;
383 if (actions & FlutterSemanticsAction::kFlutterSemanticsActionTap) {
384 node_data.AddAction(ax::mojom::Action::kDoDefault);
386 if (actions & FlutterSemanticsAction::kFlutterSemanticsActionScrollLeft) {
387 node_data.AddAction(ax::mojom::Action::kScrollLeft);
389 if (actions & FlutterSemanticsAction::kFlutterSemanticsActionScrollRight) {
390 node_data.AddAction(ax::mojom::Action::kScrollRight);
392 if (actions & FlutterSemanticsAction::kFlutterSemanticsActionScrollUp) {
393 node_data.AddAction(ax::mojom::Action::kScrollUp);
395 if (actions & FlutterSemanticsAction::kFlutterSemanticsActionScrollDown) {
396 node_data.AddAction(ax::mojom::Action::kScrollDown);
398 if (actions & FlutterSemanticsAction::kFlutterSemanticsActionIncrease) {
399 node_data.AddAction(ax::mojom::Action::kIncrement);
401 if (actions & FlutterSemanticsAction::kFlutterSemanticsActionDecrease) {
402 node_data.AddAction(ax::mojom::Action::kDecrement);
405 node_data.AddAction(ax::mojom::Action::kScrollToMakeVisible);
407 if (actions & FlutterSemanticsAction::kFlutterSemanticsActionSetSelection) {
408 node_data.AddAction(ax::mojom::Action::kSetSelection);
410 if (actions & FlutterSemanticsAction::
411 kFlutterSemanticsActionDidGainAccessibilityFocus) {
412 node_data.AddAction(ax::mojom::Action::kSetAccessibilityFocus);
414 if (actions & FlutterSemanticsAction::
415 kFlutterSemanticsActionDidLoseAccessibilityFocus) {
416 node_data.AddAction(ax::mojom::Action::kClearAccessibilityFocus);
418 if (actions & FlutterSemanticsAction::kFlutterSemanticsActionCustomAction) {
419 node_data.AddAction(ax::mojom::Action::kCustomAction);
423 void AccessibilityBridge::SetBooleanAttributesFromFlutterUpdate(
424 ui::AXNodeData& node_data,
425 const SemanticsNode& node) {
426 FlutterSemanticsAction actions = node.actions;
427 FlutterSemanticsFlag
flags = node.flags;
428 node_data.AddBoolAttribute(ax::mojom::BoolAttribute::kScrollable,
430 node_data.AddBoolAttribute(
431 ax::mojom::BoolAttribute::kClickable,
432 actions & FlutterSemanticsAction::kFlutterSemanticsActionTap);
434 node_data.AddBoolAttribute(ax::mojom::BoolAttribute::kClipsChildren,
435 !node.children_in_traversal_order.empty());
436 node_data.AddBoolAttribute(
437 ax::mojom::BoolAttribute::kSelected,
438 flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsSelected);
439 node_data.AddBoolAttribute(
440 ax::mojom::BoolAttribute::kEditableRoot,
441 flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField &&
442 (
flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsReadOnly) == 0);
447 node_data.AddBoolAttribute(ax::mojom::BoolAttribute::kIsLineBreakingObject,
451 void AccessibilityBridge::SetIntAttributesFromFlutterUpdate(
452 ui::AXNodeData& node_data,
453 const SemanticsNode& node) {
454 FlutterSemanticsFlag
flags = node.flags;
455 node_data.AddIntAttribute(ax::mojom::IntAttribute::kTextDirection,
456 node.text_direction);
458 int sel_start = node.text_selection_base;
459 int sel_end = node.text_selection_extent;
460 if (
flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField &&
461 (
flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsReadOnly) == 0 &&
462 !node.value.empty()) {
464 sel_start = sel_start == -1 ? node.value.length() : sel_start;
465 sel_end = sel_end == -1 ? node.value.length() : sel_end;
467 node_data.AddIntAttribute(ax::mojom::IntAttribute::kTextSelStart, sel_start);
468 node_data.AddIntAttribute(ax::mojom::IntAttribute::kTextSelEnd, sel_end);
470 if (node_data.role == ax::mojom::Role::kRadioButton ||
471 node_data.role == ax::mojom::Role::kCheckBox) {
472 node_data.AddIntAttribute(
473 ax::mojom::IntAttribute::kCheckedState,
474 static_cast<int32_t
>(
475 flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsCheckStateMixed
476 ? ax::mojom::CheckedState::kMixed
477 :
flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsChecked
478 ? ax::mojom::CheckedState::kTrue
479 : ax::mojom::CheckedState::kFalse));
480 }
else if (node_data.role == ax::mojom::Role::kSwitch) {
481 node_data.AddIntAttribute(
482 ax::mojom::IntAttribute::kCheckedState,
483 static_cast<int32_t
>(
484 flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsToggled
485 ? ax::mojom::CheckedState::kTrue
486 : ax::mojom::CheckedState::kFalse));
490 void AccessibilityBridge::SetIntListAttributesFromFlutterUpdate(
491 ui::AXNodeData& node_data,
492 const SemanticsNode& node) {
493 FlutterSemanticsAction actions = node.actions;
494 if (actions & FlutterSemanticsAction::kFlutterSemanticsActionCustomAction) {
495 std::vector<int32_t> custom_action_ids;
496 for (
size_t i = 0;
i < node.custom_accessibility_actions.size();
i++) {
497 custom_action_ids.push_back(node.custom_accessibility_actions[
i]);
499 node_data.AddIntListAttribute(ax::mojom::IntListAttribute::kCustomActionIds,
504 void AccessibilityBridge::SetStringListAttributesFromFlutterUpdate(
505 ui::AXNodeData& node_data,
506 const SemanticsNode& node) {
507 FlutterSemanticsAction actions = node.actions;
508 if (actions & FlutterSemanticsAction::kFlutterSemanticsActionCustomAction) {
509 std::vector<std::string> custom_action_description;
510 for (
size_t i = 0;
i < node.custom_accessibility_actions.size();
i++) {
511 auto iter = pending_semantics_custom_action_updates_.find(
512 node.custom_accessibility_actions[
i]);
513 BASE_DCHECK(iter != pending_semantics_custom_action_updates_.end());
514 custom_action_description.push_back(iter->second.label);
516 node_data.AddStringListAttribute(
517 ax::mojom::StringListAttribute::kCustomActionDescriptions,
518 custom_action_description);
522 void AccessibilityBridge::SetNameFromFlutterUpdate(ui::AXNodeData& node_data,
523 const SemanticsNode& node) {
524 node_data.SetName(node.label);
527 void AccessibilityBridge::SetValueFromFlutterUpdate(ui::AXNodeData& node_data,
528 const SemanticsNode& node) {
529 node_data.SetValue(node.value);
532 void AccessibilityBridge::SetTooltipFromFlutterUpdate(
533 ui::AXNodeData& node_data,
534 const SemanticsNode& node) {
535 node_data.SetTooltip(node.tooltip);
538 void AccessibilityBridge::SetTreeData(
const SemanticsNode& node,
539 ui::AXTreeUpdate& tree_update) {
540 FlutterSemanticsFlag
flags = node.flags;
545 if (
flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField &&
546 flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsFocused) {
547 if (node.text_selection_base != -1) {
548 tree_update.tree_data.sel_anchor_object_id = node.id;
549 tree_update.tree_data.sel_anchor_offset = node.text_selection_base;
550 tree_update.tree_data.sel_focus_object_id = node.id;
551 tree_update.tree_data.sel_focus_offset = node.text_selection_extent;
552 tree_update.has_tree_data =
true;
553 }
else if (tree_update.tree_data.sel_anchor_object_id == node.id) {
554 tree_update.tree_data.sel_anchor_object_id = ui::AXNode::kInvalidAXID;
555 tree_update.tree_data.sel_anchor_offset = -1;
556 tree_update.tree_data.sel_focus_object_id = ui::AXNode::kInvalidAXID;
557 tree_update.tree_data.sel_focus_offset = -1;
558 tree_update.has_tree_data =
true;
562 if (
flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsFocused &&
563 tree_update.tree_data.focus_id != node.id) {
564 tree_update.tree_data.focus_id = node.id;
565 tree_update.has_tree_data =
true;
566 }
else if ((
flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsFocused) ==
568 tree_update.tree_data.focus_id == node.id) {
569 tree_update.tree_data.focus_id = ui::AXNode::kInvalidAXID;
570 tree_update.has_tree_data =
true;
574 AccessibilityBridge::SemanticsNode
575 AccessibilityBridge::FromFlutterSemanticsNode(
576 const FlutterSemanticsNode2& flutter_node) {
578 result.id = flutter_node.id;
579 result.flags = flutter_node.flags;
580 result.actions = flutter_node.actions;
581 result.text_selection_base = flutter_node.text_selection_base;
582 result.text_selection_extent = flutter_node.text_selection_extent;
583 result.scroll_child_count = flutter_node.scroll_child_count;
584 result.scroll_index = flutter_node.scroll_index;
585 result.scroll_position = flutter_node.scroll_position;
586 result.scroll_extent_max = flutter_node.scroll_extent_max;
587 result.scroll_extent_min = flutter_node.scroll_extent_min;
588 result.elevation = flutter_node.elevation;
589 result.thickness = flutter_node.thickness;
590 if (flutter_node.label) {
591 result.label = std::string(flutter_node.label);
593 if (flutter_node.hint) {
594 result.hint = std::string(flutter_node.hint);
596 if (flutter_node.value) {
597 result.value = std::string(flutter_node.value);
599 if (flutter_node.increased_value) {
600 result.increased_value = std::string(flutter_node.increased_value);
602 if (flutter_node.decreased_value) {
603 result.decreased_value = std::string(flutter_node.decreased_value);
605 if (flutter_node.tooltip) {
606 result.tooltip = std::string(flutter_node.tooltip);
608 result.text_direction = flutter_node.text_direction;
609 result.rect = flutter_node.rect;
610 result.transform = flutter_node.transform;
611 if (flutter_node.child_count > 0) {
612 result.children_in_traversal_order = std::vector<int32_t>(
613 flutter_node.children_in_traversal_order,
614 flutter_node.children_in_traversal_order + flutter_node.child_count);
616 if (flutter_node.custom_accessibility_actions_count > 0) {
617 result.custom_accessibility_actions = std::vector<int32_t>(
618 flutter_node.custom_accessibility_actions,
619 flutter_node.custom_accessibility_actions +
620 flutter_node.custom_accessibility_actions_count);
625 AccessibilityBridge::SemanticsCustomAction
626 AccessibilityBridge::FromFlutterSemanticsCustomAction(
627 const FlutterSemanticsCustomAction2& flutter_custom_action) {
628 SemanticsCustomAction
result;
629 result.id = flutter_custom_action.id;
630 result.override_action = flutter_custom_action.override_action;
631 if (flutter_custom_action.label) {
632 result.label = std::string(flutter_custom_action.label);
634 if (flutter_custom_action.hint) {
635 result.hint = std::string(flutter_custom_action.hint);
641 if (last_focused_id_ != node_id) {
642 auto last_focused_child =
644 if (!last_focused_child.expired()) {
647 FlutterSemanticsAction::
648 kFlutterSemanticsActionDidLoseAccessibilityFocus,
651 last_focused_id_ = node_id;
656 return last_focused_id_;
659 gfx::NativeViewAccessible AccessibilityBridge::GetNativeAccessibleFromId(
662 if (!platform_node_delegate) {
665 return platform_node_delegate->GetNativeViewAccessible();
668 gfx::RectF AccessibilityBridge::RelativeToGlobalBounds(
const ui::AXNode* node,
671 return tree_->RelativeToTreeBounds(node, gfx::RectF(), &offscreen,
676 ui::AXTreeID tree_id,
677 ui::AXNode::AXID node_id)
const {
682 ui::AXNode::AXID node_id)
const {
683 return tree_->GetFromId(node_id);
687 return tree_->GetAXTreeID();
691 return ui::AXTreeIDUnknown();
695 return tree_->root();
707 const ui::AXNode::AXID node_id)
const {
709 auto platform_delegate = platform_delegate_weak.lock();
710 if (!platform_delegate) {
713 return platform_delegate->GetPlatformNode();
717 const ui::AXNode& node)
const {