Flutter Windows Embedder
accessibility_bridge.h
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #ifndef FLUTTER_SHELL_PLATFORM_COMMON_ACCESSIBILITY_BRIDGE_H_
6 #define FLUTTER_SHELL_PLATFORM_COMMON_ACCESSIBILITY_BRIDGE_H_
7 
8 #include <unordered_map>
9 
10 #include "flutter/fml/mapping.h"
11 #include "flutter/shell/platform/embedder/embedder.h"
12 
13 #include "flutter/third_party/accessibility/ax/ax_event_generator.h"
14 #include "flutter/third_party/accessibility/ax/ax_tree.h"
15 #include "flutter/third_party/accessibility/ax/ax_tree_observer.h"
16 #include "flutter/third_party/accessibility/ax/platform/ax_platform_node_delegate.h"
17 #include "flutter/third_party/accessibility/ax/platform/ax_platform_tree_manager.h"
18 
20 
21 namespace flutter {
22 
23 //------------------------------------------------------------------------------
24 /// Use this class to maintain an accessibility tree. This class consumes
25 /// semantics updates from the embedder API and produces an accessibility tree
26 /// in the native format.
27 ///
28 /// The bridge creates an AXTree to hold the semantics data that comes from
29 /// Flutter semantics updates. The tree holds AXNode[s] which contain the
30 /// semantics information for semantics node. The AXTree resemble the Flutter
31 /// semantics tree in the Flutter framework. The bridge also uses
32 /// FlutterPlatformNodeDelegate to wrap each AXNode in order to provide
33 /// an accessibility tree in the native format.
34 ///
35 /// To use this class, one must subclass this class and provide their own
36 /// implementation of FlutterPlatformNodeDelegate.
37 ///
38 /// AccessibilityBridge must be created as a shared_ptr, since some methods
39 /// acquires its weak_ptr.
41  : public std::enable_shared_from_this<AccessibilityBridge>,
43  public ui::AXPlatformTreeManager,
44  private ui::AXTreeObserver {
45  public:
46  //-----------------------------------------------------------------------------
47  /// @brief Creates a new instance of a accessibility bridge.
49  virtual ~AccessibilityBridge();
50 
51  //------------------------------------------------------------------------------
52  /// @brief Adds a semantics node update to the pending semantics update.
53  /// Calling this method alone will NOT update the semantics tree.
54  /// To flush the pending updates, call the CommitUpdates().
55  ///
56  /// @param[in] node A reference to the semantics node update.
57  void AddFlutterSemanticsNodeUpdate(const FlutterSemanticsNode2& node);
58 
59  //------------------------------------------------------------------------------
60  /// @brief Adds a custom semantics action update to the pending semantics
61  /// update. Calling this method alone will NOT update the
62  /// semantics tree. To flush the pending updates, call the
63  /// CommitUpdates().
64  ///
65  /// @param[in] action A reference to the custom semantics action
66  /// update.
68  const FlutterSemanticsCustomAction2& action);
69 
70  //------------------------------------------------------------------------------
71  /// @brief Flushes the pending updates and applies them to this
72  /// accessibility bridge. Calling this with no pending updates
73  /// does nothing, and callers should call this method at the end
74  /// of an atomic batch to avoid leaving the tree in a unstable
75  /// state. For example if a node reparents from A to B, callers
76  /// should only call this method when both removal from A and
77  /// addition to B are in the pending updates.
78  void CommitUpdates();
79 
80  //------------------------------------------------------------------------------
81  /// @brief Get the flutter platform node delegate with the given id from
82  /// this accessibility bridge. Returns expired weak_ptr if the
83  /// delegate associated with the id does not exist or has been
84  /// removed from the accessibility tree.
85  ///
86  /// @param[in] id The id of the flutter accessibility node you want
87  /// to retrieve.
88  std::weak_ptr<FlutterPlatformNodeDelegate>
90 
91  //------------------------------------------------------------------------------
92  /// @brief Get the ax tree data from this accessibility bridge. The tree
93  /// data contains information such as the id of the node that
94  /// has the keyboard focus or the text selection range.
95  const ui::AXTreeData& GetAXTreeData() const;
96 
97  //------------------------------------------------------------------------------
98  /// @brief Gets all pending accessibility events generated during
99  /// semantics updates. This is useful when deciding how to handle
100  /// events in AccessibilityBridgeDelegate::OnAccessibilityEvent in
101  /// case one may decide to handle an event differently based on
102  /// all pending events.
103  const std::vector<ui::AXEventGenerator::TargetedEvent> GetPendingEvents()
104  const;
105 
106  // |AXTreeManager|
107  ui::AXNode* GetNodeFromTree(const ui::AXTreeID tree_id,
108  const ui::AXNode::AXID node_id) const override;
109 
110  // |AXTreeManager|
111  ui::AXNode* GetNodeFromTree(const ui::AXNode::AXID node_id) const override;
112 
113  // |AXTreeManager|
114  ui::AXTreeID GetTreeID() const override;
115 
116  // |AXTreeManager|
117  ui::AXTreeID GetParentTreeID() const override;
118 
119  // |AXTreeManager|
120  ui::AXNode* GetRootAsAXNode() const override;
121 
122  // |AXTreeManager|
123  ui::AXNode* GetParentNodeFromParentTreeAsAXNode() const override;
124 
125  // |AXTreeManager|
126  ui::AXTree* GetTree() const override;
127 
128  // |AXPlatformTreeManager|
129  ui::AXPlatformNode* GetPlatformNodeFromTree(
130  const ui::AXNode::AXID node_id) const override;
131 
132  // |AXPlatformTreeManager|
133  ui::AXPlatformNode* GetPlatformNodeFromTree(
134  const ui::AXNode& node) const override;
135 
136  // |AXPlatformTreeManager|
137  ui::AXPlatformNodeDelegate* RootDelegate() const override;
138 
139  protected:
140  //---------------------------------------------------------------------------
141  /// @brief Handle accessibility events generated due to accessibility
142  /// tree changes. These events are needed to be sent to native
143  /// accessibility system. See ui::AXEventGenerator::Event for
144  /// possible events.
145  ///
146  /// @param[in] targeted_event The object that contains both the
147  /// generated event and the event target.
148  virtual void OnAccessibilityEvent(
149  ui::AXEventGenerator::TargetedEvent targeted_event) = 0;
150 
151  //---------------------------------------------------------------------------
152  /// @brief Creates a platform specific FlutterPlatformNodeDelegate.
153  /// Ownership passes to the caller. This method will be called
154  /// whenever a new AXNode is created in AXTree. Each platform
155  /// needs to implement this method in order to inject its
156  /// subclass into the accessibility bridge.
157  virtual std::shared_ptr<FlutterPlatformNodeDelegate>
159 
160  private:
161  // See FlutterSemanticsNode in embedder.h
162  typedef struct {
163  int32_t id;
164  FlutterSemanticsFlag flags;
165  FlutterSemanticsAction actions;
166  int32_t text_selection_base;
167  int32_t text_selection_extent;
168  int32_t scroll_child_count;
169  int32_t scroll_index;
170  double scroll_position;
171  double scroll_extent_max;
172  double scroll_extent_min;
173  double elevation;
174  double thickness;
175  std::string label;
176  std::string hint;
177  std::string value;
178  std::string increased_value;
179  std::string decreased_value;
180  std::string tooltip;
181  FlutterTextDirection text_direction;
182  FlutterRect rect;
183  FlutterTransformation transform;
184  std::vector<int32_t> children_in_traversal_order;
185  std::vector<int32_t> custom_accessibility_actions;
186  } SemanticsNode;
187 
188  // See FlutterSemanticsCustomAction in embedder.h
189  typedef struct {
190  int32_t id;
191  FlutterSemanticsAction override_action;
192  std::string label;
193  std::string hint;
194  } SemanticsCustomAction;
195 
196  std::unordered_map<AccessibilityNodeId,
197  std::shared_ptr<FlutterPlatformNodeDelegate>>
198  id_wrapper_map_;
199  std::unique_ptr<ui::AXTree> tree_;
200  ui::AXEventGenerator event_generator_;
201  std::unordered_map<int32_t, SemanticsNode> pending_semantics_node_updates_;
202  std::unordered_map<int32_t, SemanticsCustomAction>
203  pending_semantics_custom_action_updates_;
204  AccessibilityNodeId last_focused_id_ = ui::AXNode::kInvalidAXID;
205 
206  void InitAXTree(const ui::AXTreeUpdate& initial_state);
207 
208  // Create an update that removes any nodes that will be reparented by
209  // pending_semantics_updates_. Returns std::nullopt if none are reparented.
210  std::optional<ui::AXTreeUpdate> CreateRemoveReparentedNodesUpdate();
211 
212  void GetSubTreeList(const SemanticsNode& target,
213  std::vector<SemanticsNode>& result);
214  void ConvertFlutterUpdate(const SemanticsNode& node,
215  ui::AXTreeUpdate& tree_update);
216  void SetRoleFromFlutterUpdate(ui::AXNodeData& node_data,
217  const SemanticsNode& node);
218  void SetStateFromFlutterUpdate(ui::AXNodeData& node_data,
219  const SemanticsNode& node);
220  void SetActionsFromFlutterUpdate(ui::AXNodeData& node_data,
221  const SemanticsNode& node);
222  void SetBooleanAttributesFromFlutterUpdate(ui::AXNodeData& node_data,
223  const SemanticsNode& node);
224  void SetIntAttributesFromFlutterUpdate(ui::AXNodeData& node_data,
225  const SemanticsNode& node);
226  void SetIntListAttributesFromFlutterUpdate(ui::AXNodeData& node_data,
227  const SemanticsNode& node);
228  void SetStringListAttributesFromFlutterUpdate(ui::AXNodeData& node_data,
229  const SemanticsNode& node);
230  void SetNameFromFlutterUpdate(ui::AXNodeData& node_data,
231  const SemanticsNode& node);
232  void SetValueFromFlutterUpdate(ui::AXNodeData& node_data,
233  const SemanticsNode& node);
234  void SetTooltipFromFlutterUpdate(ui::AXNodeData& node_data,
235  const SemanticsNode& node);
236  void SetTreeData(const SemanticsNode& node, ui::AXTreeUpdate& tree_update);
237  SemanticsNode FromFlutterSemanticsNode(
238  const FlutterSemanticsNode2& flutter_node);
239  SemanticsCustomAction FromFlutterSemanticsCustomAction(
240  const FlutterSemanticsCustomAction2& flutter_custom_action);
241 
242  // |AXTreeObserver|
243  void OnNodeWillBeDeleted(ui::AXTree* tree, ui::AXNode* node) override;
244 
245  // |AXTreeObserver|
246  void OnSubtreeWillBeDeleted(ui::AXTree* tree, ui::AXNode* node) override;
247 
248  // |AXTreeObserver|
249  void OnNodeCreated(ui::AXTree* tree, ui::AXNode* node) override;
250 
251  // |AXTreeObserver|
252  void OnNodeDeleted(ui::AXTree* tree, AccessibilityNodeId node_id) override;
253 
254  // |AXTreeObserver|
255  void OnNodeReparented(ui::AXTree* tree, ui::AXNode* node) override;
256 
257  // |AXTreeObserver|
258  void OnRoleChanged(ui::AXTree* tree,
259  ui::AXNode* node,
260  ax::mojom::Role old_role,
261  ax::mojom::Role new_role) override;
262 
263  // |AXTreeObserver|
264  void OnAtomicUpdateFinished(
265  ui::AXTree* tree,
266  bool root_changed,
267  const std::vector<ui::AXTreeObserver::Change>& changes) override;
268 
269  // |FlutterPlatformNodeDelegate::OwnerBridge|
270  void SetLastFocusedId(AccessibilityNodeId node_id) override;
271 
272  // |FlutterPlatformNodeDelegate::OwnerBridge|
273  AccessibilityNodeId GetLastFocusedId() override;
274 
275  // |FlutterPlatformNodeDelegate::OwnerBridge|
276  gfx::NativeViewAccessible GetNativeAccessibleFromId(
277  AccessibilityNodeId id) override;
278 
279  // |FlutterPlatformNodeDelegate::OwnerBridge|
280  gfx::RectF RelativeToGlobalBounds(const ui::AXNode* node,
281  bool& offscreen,
282  bool clip_bounds) override;
283 
284  BASE_DISALLOW_COPY_AND_ASSIGN(AccessibilityBridge);
285 };
286 
287 } // namespace flutter
288 
289 #endif // FLUTTER_SHELL_PLATFORM_COMMON_ACCESSIBILITY_BRIDGE_H_
flutter::AccessibilityBridge::GetTreeID
ui::AXTreeID GetTreeID() const override
Definition: accessibility_bridge.cc:686
flutter::AccessibilityBridge::OnAccessibilityEvent
virtual void OnAccessibilityEvent(ui::AXEventGenerator::TargetedEvent targeted_event)=0
Handle accessibility events generated due to accessibility tree changes. These events are needed to b...
flutter::AccessibilityBridge::GetPlatformNodeFromTree
ui::AXPlatformNode * GetPlatformNodeFromTree(const ui::AXNode::AXID node_id) const override
Definition: accessibility_bridge.cc:706
flutter::AccessibilityBridge::GetTree
ui::AXTree * GetTree() const override
Definition: accessibility_bridge.cc:702
flutter::AccessibilityBridge::GetRootAsAXNode
ui::AXNode * GetRootAsAXNode() const override
Definition: accessibility_bridge.cc:694
flutter::AccessibilityBridge::AddFlutterSemanticsCustomActionUpdate
void AddFlutterSemanticsCustomActionUpdate(const FlutterSemanticsCustomAction2 &action)
Adds a custom semantics action update to the pending semantics update. Calling this method alone will...
Definition: accessibility_bridge.cc:44
flutter::AccessibilityBridge::GetParentNodeFromParentTreeAsAXNode
ui::AXNode * GetParentNodeFromParentTreeAsAXNode() const override
Definition: accessibility_bridge.cc:698
flutter::AccessibilityBridge
Definition: accessibility_bridge.h:40
flutter::AccessibilityBridge::GetFlutterPlatformNodeDelegateFromID
std::weak_ptr< FlutterPlatformNodeDelegate > GetFlutterPlatformNodeDelegateFromID(AccessibilityNodeId id) const
Get the flutter platform node delegate with the given id from this accessibility bridge....
Definition: accessibility_bridge.cc:131
flutter::AccessibilityBridge::GetParentTreeID
ui::AXTreeID GetParentTreeID() const override
Definition: accessibility_bridge.cc:690
flutter::AccessibilityBridge::AccessibilityBridge
AccessibilityBridge()
Creates a new instance of a accessibility bridge.
Definition: accessibility_bridge.cc:23
flutter::AccessibilityNodeId
ui::AXNode::AXID AccessibilityNodeId
Definition: flutter_platform_node_delegate.h:15
flutter
Definition: accessibility_bridge_windows.cc:11
flutter_platform_node_delegate.h
flutter::AccessibilityBridge::CreateFlutterPlatformNodeDelegate
virtual std::shared_ptr< FlutterPlatformNodeDelegate > CreateFlutterPlatformNodeDelegate()=0
Creates a platform specific FlutterPlatformNodeDelegate. Ownership passes to the caller....
flutter::AccessibilityBridge::CommitUpdates
void CommitUpdates()
Flushes the pending updates and applies them to this accessibility bridge. Calling this with no pendi...
Definition: accessibility_bridge.cc:50
flutter::AccessibilityBridge::GetAXTreeData
const ui::AXTreeData & GetAXTreeData() const
Get the ax tree data from this accessibility bridge. The tree data contains information such as the i...
Definition: accessibility_bridge.cc:141
flutter::AccessibilityBridge::GetPendingEvents
const std::vector< ui::AXEventGenerator::TargetedEvent > GetPendingEvents() const
Gets all pending accessibility events generated during semantics updates. This is useful when decidin...
Definition: accessibility_bridge.cc:146
flutter::AccessibilityBridge::GetNodeFromTree
ui::AXNode * GetNodeFromTree(const ui::AXTreeID tree_id, const ui::AXNode::AXID node_id) const override
Definition: accessibility_bridge.cc:675
action
int action
Definition: keyboard_key_handler_unittests.cc:116
flutter::AccessibilityBridge::RootDelegate
ui::AXPlatformNodeDelegate * RootDelegate() const override
Definition: accessibility_bridge.cc:721
flutter::FlutterPlatformNodeDelegate::OwnerBridge
Definition: flutter_platform_node_delegate.h:38
flutter::AccessibilityBridge::AddFlutterSemanticsNodeUpdate
void AddFlutterSemanticsNodeUpdate(const FlutterSemanticsNode2 &node)
Adds a semantics node update to the pending semantics update. Calling this method alone will NOT upda...
Definition: accessibility_bridge.cc:39
flutter::AccessibilityBridge::~AccessibilityBridge
virtual ~AccessibilityBridge()
Definition: accessibility_bridge.cc:34