10 #include "flutter/third_party/accessibility/ax/ax_tree_manager_map.h"
11 #include "flutter/third_party/accessibility/ax/ax_tree_update.h"
12 #include "flutter/third_party/accessibility/base/logging.h"
17 FlutterSemanticsAction::kFlutterSemanticsActionScrollLeft |
18 FlutterSemanticsAction::kFlutterSemanticsActionScrollRight |
19 FlutterSemanticsAction::kFlutterSemanticsActionScrollUp |
20 FlutterSemanticsAction::kFlutterSemanticsActionScrollDown;
24 : tree_(std::make_unique<ui::AXTree>()) {
25 event_generator_.SetTree(tree_.get());
26 tree_->AddObserver(
static_cast<ui::AXTreeObserver*
>(
this));
27 ui::AXTreeData data = tree_->data();
28 data.tree_id = ui::AXTreeID::CreateNewAXTreeID();
29 tree_->UpdateData(data);
30 ui::AXTreeManagerMap::GetInstance().AddTreeManager(tree_->GetAXTreeID(),
35 event_generator_.ReleaseTree();
36 tree_->RemoveObserver(
static_cast<ui::AXTreeObserver*
>(
this));
40 const FlutterSemanticsNode2& node) {
41 pending_semantics_node_updates_[node.id] = FromFlutterSemanticsNode(node);
45 const FlutterSemanticsCustomAction2& action) {
46 pending_semantics_custom_action_updates_[action.id] =
47 FromFlutterSemanticsCustomAction(action);
58 std::optional<ui::AXTreeUpdate> remove_reparented =
59 CreateRemoveReparentedNodesUpdate();
60 if (remove_reparented.has_value()) {
61 tree_->Unserialize(remove_reparented.value());
63 std::string error = tree_->error();
65 FML_LOG(ERROR) <<
"Failed to update ui::AXTree, error: " << error;
73 ui::AXTreeUpdate update{.tree_data = tree_->data()};
83 std::vector<std::vector<SemanticsNode>> results;
84 while (!pending_semantics_node_updates_.empty()) {
85 auto begin = pending_semantics_node_updates_.begin();
86 SemanticsNode target = begin->second;
87 std::vector<SemanticsNode> sub_tree_list;
88 GetSubTreeList(target, sub_tree_list);
89 results.push_back(sub_tree_list);
90 pending_semantics_node_updates_.erase(begin);
93 for (
size_t i = results.size(); i > 0; i--) {
94 for (
const SemanticsNode& node : results[i - 1]) {
95 ConvertFlutterUpdate(node, update);
102 if (!results.empty() &&
GetRootAsAXNode()->
id() == ui::AXNode::kInvalidAXID) {
103 FML_DCHECK(!results.back().empty());
105 update.root_id = results.back().front().id;
108 tree_->Unserialize(update);
109 pending_semantics_node_updates_.clear();
110 pending_semantics_custom_action_updates_.clear();
112 std::string error = tree_->error();
113 if (!error.empty()) {
114 FML_LOG(ERROR) <<
"Failed to update ui::AXTree, error: " << error;
118 for (
const auto& targeted_event : event_generator_) {
121 if (event_target.expired()) {
127 event_generator_.ClearEvents();
130 std::weak_ptr<FlutterPlatformNodeDelegate>
133 const auto iter = id_wrapper_map_.find(
id);
134 if (iter != id_wrapper_map_.end()) {
138 return std::weak_ptr<FlutterPlatformNodeDelegate>();
142 return tree_->data();
145 const std::vector<ui::AXEventGenerator::TargetedEvent>
147 std::vector<ui::AXEventGenerator::TargetedEvent> result(
148 event_generator_.begin(), event_generator_.end());
152 void AccessibilityBridge::OnNodeWillBeDeleted(ui::AXTree* tree,
155 void AccessibilityBridge::OnSubtreeWillBeDeleted(ui::AXTree* tree,
158 void AccessibilityBridge::OnNodeReparented(ui::AXTree* tree, ui::AXNode* node) {
161 void AccessibilityBridge::OnRoleChanged(ui::AXTree* tree,
163 ax::mojom::Role old_role,
164 ax::mojom::Role new_role) {}
166 void AccessibilityBridge::OnNodeDataChanged(
168 const ui::AXNodeData& old_node_data,
169 const ui::AXNodeData& new_node_data) {
173 platform_view->NodeDataChanged(old_node_data, new_node_data);
177 void AccessibilityBridge::OnNodeCreated(ui::AXTree* tree, ui::AXNode* node) {
180 id_wrapper_map_[node->id()]->Init(
181 std::static_pointer_cast<FlutterPlatformNodeDelegate::OwnerBridge>(
186 void AccessibilityBridge::OnNodeDeleted(ui::AXTree* tree,
188 BASE_DCHECK(node_id != ui::AXNode::kInvalidAXID);
189 if (id_wrapper_map_.find(node_id) != id_wrapper_map_.end()) {
190 id_wrapper_map_.erase(node_id);
194 void AccessibilityBridge::OnAtomicUpdateFinished(
197 const std::vector<ui::AXTreeObserver::Change>& changes) {
201 for (
const auto& change : changes) {
202 ui::AXNode* node = change.node;
203 const ui::AXNodeData& data = node->data();
205 if (node->parent()) {
206 offset_container_id = node->parent()->id();
208 node->SetLocation(offset_container_id, data.relative_bounds.bounds,
209 data.relative_bounds.transform.get());
213 std::optional<ui::AXTreeUpdate>
214 AccessibilityBridge::CreateRemoveReparentedNodesUpdate() {
215 std::unordered_map<int32_t, ui::AXNodeData> updates;
217 for (
const auto& node_update : pending_semantics_node_updates_) {
218 for (int32_t child_id : node_update.second.children_in_traversal_order) {
220 ui::AXNode* child = tree_->GetFromId(child_id);
226 assert(child->parent());
229 if (child->parent()->id() == node_update.second.id) {
235 assert(pending_semantics_node_updates_.find(child_id) !=
236 pending_semantics_node_updates_.end());
239 int32_t parent_id = child->parent()->id();
240 if (updates.find(parent_id) == updates.end()) {
241 updates[parent_id] = tree_->GetFromId(parent_id)->data();
244 ui::AXNodeData* parent = &updates[parent_id];
245 auto iter = std::find(parent->child_ids.begin(), parent->child_ids.end(),
248 assert(iter != parent->child_ids.end());
249 parent->child_ids.erase(iter);
253 if (updates.empty()) {
257 ui::AXTreeUpdate update{
258 .tree_data = tree_->data(),
259 .nodes = std::vector<ui::AXNodeData>(),
262 for (std::pair<int32_t, ui::AXNodeData> data : updates) {
263 update.nodes.push_back(std::move(data.second));
270 void AccessibilityBridge::GetSubTreeList(
const SemanticsNode& target,
271 std::vector<SemanticsNode>& result) {
272 result.push_back(target);
273 for (int32_t child : target.children_in_traversal_order) {
274 auto iter = pending_semantics_node_updates_.find(child);
275 if (iter != pending_semantics_node_updates_.end()) {
276 SemanticsNode node = iter->second;
277 GetSubTreeList(node, result);
278 pending_semantics_node_updates_.erase(iter);
283 void AccessibilityBridge::ConvertFlutterUpdate(
const SemanticsNode& node,
284 ui::AXTreeUpdate& tree_update) {
285 ui::AXNodeData node_data;
286 node_data.id = node.id;
287 SetRoleFromFlutterUpdate(node_data, node);
288 SetStateFromFlutterUpdate(node_data, node);
289 SetActionsFromFlutterUpdate(node_data, node);
290 SetBooleanAttributesFromFlutterUpdate(node_data, node);
291 SetIntAttributesFromFlutterUpdate(node_data, node);
292 SetIntListAttributesFromFlutterUpdate(node_data, node);
293 SetStringListAttributesFromFlutterUpdate(node_data, node);
294 SetNameFromFlutterUpdate(node_data, node);
295 SetValueFromFlutterUpdate(node_data, node);
296 SetTooltipFromFlutterUpdate(node_data, node);
297 node_data.relative_bounds.bounds.SetRect(node.rect.left, node.rect.top,
298 node.rect.right - node.rect.left,
299 node.rect.bottom - node.rect.top);
300 node_data.relative_bounds.transform = std::make_unique<gfx::Transform>(
301 node.transform.scaleX, node.transform.skewX, node.transform.transX, 0,
302 node.transform.skewY, node.transform.scaleY, node.transform.transY, 0,
303 node.transform.pers0, node.transform.pers1, node.transform.pers2, 0, 0, 0,
305 for (
auto child : node.children_in_traversal_order) {
306 node_data.child_ids.push_back(child);
308 SetTreeData(node, tree_update);
309 tree_update.nodes.push_back(node_data);
312 void AccessibilityBridge::SetRoleFromFlutterUpdate(ui::AXNodeData& node_data,
313 const SemanticsNode& node) {
314 FlutterSemanticsFlag flags = node.flags;
315 if (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsButton) {
316 node_data.role = ax::mojom::Role::kButton;
319 if (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField &&
320 !(flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsReadOnly)) {
321 node_data.role = ax::mojom::Role::kTextField;
324 if (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsHeader) {
325 node_data.role = ax::mojom::Role::kHeader;
328 if (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsImage) {
329 node_data.role = ax::mojom::Role::kImage;
332 if (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsLink) {
333 node_data.role = ax::mojom::Role::kLink;
337 if (flags & kFlutterSemanticsFlagIsInMutuallyExclusiveGroup &&
338 flags & kFlutterSemanticsFlagHasCheckedState) {
339 node_data.role = ax::mojom::Role::kRadioButton;
342 if (flags & kFlutterSemanticsFlagHasCheckedState) {
343 node_data.role = ax::mojom::Role::kCheckBox;
346 if (flags & kFlutterSemanticsFlagHasToggledState) {
347 node_data.role = ax::mojom::Role::kSwitch;
350 if (flags & kFlutterSemanticsFlagIsSlider) {
351 node_data.role = ax::mojom::Role::kSlider;
356 if (node.children_in_traversal_order.empty()) {
357 node_data.role = ax::mojom::Role::kStaticText;
359 node_data.role = ax::mojom::Role::kGroup;
363 void AccessibilityBridge::SetStateFromFlutterUpdate(ui::AXNodeData& node_data,
364 const SemanticsNode& node) {
365 FlutterSemanticsFlag flags = node.flags;
366 FlutterSemanticsAction actions = node.actions;
367 if (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagHasExpandedState &&
368 flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsExpanded) {
369 node_data.AddState(ax::mojom::State::kExpanded);
371 FlutterSemanticsFlag::kFlutterSemanticsFlagHasExpandedState) {
372 node_data.AddState(ax::mojom::State::kCollapsed);
374 if (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField &&
375 (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsReadOnly) == 0) {
376 node_data.AddState(ax::mojom::State::kEditable);
378 if (node_data.role == ax::mojom::Role::kStaticText &&
380 node.label.empty() && node.hint.empty()) {
381 node_data.AddState(ax::mojom::State::kIgnored);
386 node_data.AddState(ax::mojom::State::kFocusable);
390 void AccessibilityBridge::SetActionsFromFlutterUpdate(
391 ui::AXNodeData& node_data,
392 const SemanticsNode& node) {
393 FlutterSemanticsAction actions = node.actions;
394 if (actions & FlutterSemanticsAction::kFlutterSemanticsActionTap) {
395 node_data.AddAction(ax::mojom::Action::kDoDefault);
397 if (actions & FlutterSemanticsAction::kFlutterSemanticsActionScrollLeft) {
398 node_data.AddAction(ax::mojom::Action::kScrollLeft);
400 if (actions & FlutterSemanticsAction::kFlutterSemanticsActionScrollRight) {
401 node_data.AddAction(ax::mojom::Action::kScrollRight);
403 if (actions & FlutterSemanticsAction::kFlutterSemanticsActionScrollUp) {
404 node_data.AddAction(ax::mojom::Action::kScrollUp);
406 if (actions & FlutterSemanticsAction::kFlutterSemanticsActionScrollDown) {
407 node_data.AddAction(ax::mojom::Action::kScrollDown);
409 if (actions & FlutterSemanticsAction::kFlutterSemanticsActionIncrease) {
410 node_data.AddAction(ax::mojom::Action::kIncrement);
412 if (actions & FlutterSemanticsAction::kFlutterSemanticsActionDecrease) {
413 node_data.AddAction(ax::mojom::Action::kDecrement);
416 node_data.AddAction(ax::mojom::Action::kScrollToMakeVisible);
418 if (actions & FlutterSemanticsAction::kFlutterSemanticsActionSetSelection) {
419 node_data.AddAction(ax::mojom::Action::kSetSelection);
421 if (actions & FlutterSemanticsAction::
422 kFlutterSemanticsActionDidGainAccessibilityFocus) {
423 node_data.AddAction(ax::mojom::Action::kSetAccessibilityFocus);
425 if (actions & FlutterSemanticsAction::
426 kFlutterSemanticsActionDidLoseAccessibilityFocus) {
427 node_data.AddAction(ax::mojom::Action::kClearAccessibilityFocus);
429 if (actions & FlutterSemanticsAction::kFlutterSemanticsActionCustomAction) {
430 node_data.AddAction(ax::mojom::Action::kCustomAction);
434 void AccessibilityBridge::SetBooleanAttributesFromFlutterUpdate(
435 ui::AXNodeData& node_data,
436 const SemanticsNode& node) {
437 FlutterSemanticsAction actions = node.actions;
438 FlutterSemanticsFlag flags = node.flags;
439 node_data.AddBoolAttribute(ax::mojom::BoolAttribute::kScrollable,
441 node_data.AddBoolAttribute(
442 ax::mojom::BoolAttribute::kClickable,
443 actions & FlutterSemanticsAction::kFlutterSemanticsActionTap);
445 node_data.AddBoolAttribute(ax::mojom::BoolAttribute::kClipsChildren,
446 !node.children_in_traversal_order.empty());
447 node_data.AddBoolAttribute(
448 ax::mojom::BoolAttribute::kSelected,
449 flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsSelected);
450 node_data.AddBoolAttribute(
451 ax::mojom::BoolAttribute::kEditableRoot,
452 flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField &&
453 (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsReadOnly) == 0);
458 node_data.AddBoolAttribute(ax::mojom::BoolAttribute::kIsLineBreakingObject,
462 void AccessibilityBridge::SetIntAttributesFromFlutterUpdate(
463 ui::AXNodeData& node_data,
464 const SemanticsNode& node) {
465 FlutterSemanticsFlag flags = node.flags;
466 node_data.AddIntAttribute(ax::mojom::IntAttribute::kTextDirection,
467 node.text_direction);
469 int sel_start = node.text_selection_base;
470 int sel_end = node.text_selection_extent;
471 if (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField &&
472 (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsReadOnly) == 0 &&
473 !node.value.empty()) {
475 sel_start = sel_start == -1 ? node.value.length() : sel_start;
476 sel_end = sel_end == -1 ? node.value.length() : sel_end;
478 node_data.AddIntAttribute(ax::mojom::IntAttribute::kTextSelStart, sel_start);
479 node_data.AddIntAttribute(ax::mojom::IntAttribute::kTextSelEnd, sel_end);
481 if (node_data.role == ax::mojom::Role::kRadioButton ||
482 node_data.role == ax::mojom::Role::kCheckBox) {
483 node_data.AddIntAttribute(
484 ax::mojom::IntAttribute::kCheckedState,
485 static_cast<int32_t
>(
486 flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsCheckStateMixed
487 ? ax::mojom::CheckedState::kMixed
488 : flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsChecked
489 ? ax::mojom::CheckedState::kTrue
490 : ax::mojom::CheckedState::kFalse));
491 }
else if (node_data.role == ax::mojom::Role::kSwitch) {
492 node_data.AddIntAttribute(
493 ax::mojom::IntAttribute::kCheckedState,
494 static_cast<int32_t
>(
495 flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsToggled
496 ? ax::mojom::CheckedState::kTrue
497 : ax::mojom::CheckedState::kFalse));
501 void AccessibilityBridge::SetIntListAttributesFromFlutterUpdate(
502 ui::AXNodeData& node_data,
503 const SemanticsNode& node) {
504 FlutterSemanticsAction actions = node.actions;
505 if (actions & FlutterSemanticsAction::kFlutterSemanticsActionCustomAction) {
506 std::vector<int32_t> custom_action_ids;
507 for (
size_t i = 0; i < node.custom_accessibility_actions.size(); i++) {
508 custom_action_ids.push_back(node.custom_accessibility_actions[i]);
510 node_data.AddIntListAttribute(ax::mojom::IntListAttribute::kCustomActionIds,
515 void AccessibilityBridge::SetStringListAttributesFromFlutterUpdate(
516 ui::AXNodeData& node_data,
517 const SemanticsNode& node) {
518 FlutterSemanticsAction actions = node.actions;
519 if (actions & FlutterSemanticsAction::kFlutterSemanticsActionCustomAction) {
520 std::vector<std::string> custom_action_description;
521 for (
size_t i = 0; i < node.custom_accessibility_actions.size(); i++) {
522 auto iter = pending_semantics_custom_action_updates_.find(
523 node.custom_accessibility_actions[i]);
524 BASE_DCHECK(iter != pending_semantics_custom_action_updates_.end());
525 custom_action_description.push_back(iter->second.label);
527 node_data.AddStringListAttribute(
528 ax::mojom::StringListAttribute::kCustomActionDescriptions,
529 custom_action_description);
533 void AccessibilityBridge::SetNameFromFlutterUpdate(ui::AXNodeData& node_data,
534 const SemanticsNode& node) {
535 node_data.SetName(node.label);
538 void AccessibilityBridge::SetValueFromFlutterUpdate(ui::AXNodeData& node_data,
539 const SemanticsNode& node) {
540 node_data.SetValue(node.value);
543 void AccessibilityBridge::SetTooltipFromFlutterUpdate(
544 ui::AXNodeData& node_data,
545 const SemanticsNode& node) {
546 node_data.SetTooltip(node.tooltip);
549 void AccessibilityBridge::SetTreeData(
const SemanticsNode& node,
550 ui::AXTreeUpdate& tree_update) {
551 FlutterSemanticsFlag flags = node.flags;
556 if (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField &&
557 flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsFocused) {
558 if (node.text_selection_base != -1) {
559 tree_update.tree_data.sel_anchor_object_id = node.id;
560 tree_update.tree_data.sel_anchor_offset = node.text_selection_base;
561 tree_update.tree_data.sel_focus_object_id = node.id;
562 tree_update.tree_data.sel_focus_offset = node.text_selection_extent;
563 tree_update.has_tree_data =
true;
564 }
else if (tree_update.tree_data.sel_anchor_object_id == node.id) {
565 tree_update.tree_data.sel_anchor_object_id = ui::AXNode::kInvalidAXID;
566 tree_update.tree_data.sel_anchor_offset = -1;
567 tree_update.tree_data.sel_focus_object_id = ui::AXNode::kInvalidAXID;
568 tree_update.tree_data.sel_focus_offset = -1;
569 tree_update.has_tree_data =
true;
573 if (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsFocused &&
574 tree_update.tree_data.focus_id != node.id) {
575 tree_update.tree_data.focus_id = node.id;
576 tree_update.has_tree_data =
true;
577 }
else if ((flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsFocused) ==
579 tree_update.tree_data.focus_id == node.id) {
580 tree_update.tree_data.focus_id = ui::AXNode::kInvalidAXID;
581 tree_update.has_tree_data =
true;
585 AccessibilityBridge::SemanticsNode
586 AccessibilityBridge::FromFlutterSemanticsNode(
587 const FlutterSemanticsNode2& flutter_node) {
588 SemanticsNode result;
589 result.id = flutter_node.id;
590 result.flags = flutter_node.flags;
591 result.actions = flutter_node.actions;
592 result.text_selection_base = flutter_node.text_selection_base;
593 result.text_selection_extent = flutter_node.text_selection_extent;
594 result.scroll_child_count = flutter_node.scroll_child_count;
595 result.scroll_index = flutter_node.scroll_index;
596 result.scroll_position = flutter_node.scroll_position;
597 result.scroll_extent_max = flutter_node.scroll_extent_max;
598 result.scroll_extent_min = flutter_node.scroll_extent_min;
599 result.elevation = flutter_node.elevation;
600 result.thickness = flutter_node.thickness;
601 if (flutter_node.label) {
602 result.label = std::string(flutter_node.label);
604 if (flutter_node.hint) {
605 result.hint = std::string(flutter_node.hint);
607 if (flutter_node.value) {
608 result.value = std::string(flutter_node.value);
610 if (flutter_node.increased_value) {
611 result.increased_value = std::string(flutter_node.increased_value);
613 if (flutter_node.decreased_value) {
614 result.decreased_value = std::string(flutter_node.decreased_value);
616 if (flutter_node.tooltip) {
617 result.tooltip = std::string(flutter_node.tooltip);
619 result.text_direction = flutter_node.text_direction;
620 result.rect = flutter_node.rect;
621 result.transform = flutter_node.transform;
622 if (flutter_node.child_count > 0) {
623 result.children_in_traversal_order = std::vector<int32_t>(
624 flutter_node.children_in_traversal_order,
625 flutter_node.children_in_traversal_order + flutter_node.child_count);
627 if (flutter_node.custom_accessibility_actions_count > 0) {
628 result.custom_accessibility_actions = std::vector<int32_t>(
629 flutter_node.custom_accessibility_actions,
630 flutter_node.custom_accessibility_actions +
631 flutter_node.custom_accessibility_actions_count);
636 AccessibilityBridge::SemanticsCustomAction
637 AccessibilityBridge::FromFlutterSemanticsCustomAction(
638 const FlutterSemanticsCustomAction2& flutter_custom_action) {
639 SemanticsCustomAction result;
640 result.id = flutter_custom_action.id;
641 result.override_action = flutter_custom_action.override_action;
642 if (flutter_custom_action.label) {
643 result.label = std::string(flutter_custom_action.label);
645 if (flutter_custom_action.hint) {
646 result.hint = std::string(flutter_custom_action.hint);
652 if (last_focused_id_ != node_id) {
653 auto last_focused_child =
655 if (!last_focused_child.expired()) {
656 DispatchAccessibilityAction(
658 FlutterSemanticsAction::
659 kFlutterSemanticsActionDidLoseAccessibilityFocus,
662 last_focused_id_ = node_id;
667 return last_focused_id_;
670 gfx::NativeViewAccessible AccessibilityBridge::GetNativeAccessibleFromId(
673 if (!platform_node_delegate) {
676 return platform_node_delegate->GetNativeViewAccessible();
679 gfx::RectF AccessibilityBridge::RelativeToGlobalBounds(
const ui::AXNode* node,
682 return tree_->RelativeToTreeBounds(node, gfx::RectF(), &offscreen,
687 ui::AXTreeID tree_id,
688 ui::AXNode::AXID node_id)
const {
693 ui::AXNode::AXID node_id)
const {
694 return tree_->GetFromId(node_id);
698 return tree_->GetAXTreeID();
702 return ui::AXTreeIDUnknown();
706 return tree_->root();
718 const ui::AXNode::AXID node_id)
const {
720 auto platform_delegate = platform_delegate_weak.lock();
721 if (!platform_delegate) {
724 return platform_delegate->GetPlatformNode();
728 const ui::AXNode& node)
const {