Flutter Windows Embedder
accessibility_bridge_windows_unittests.cc
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "flutter/shell/platform/windows/accessibility_bridge_windows.h"
6 
7 #include <comdef.h>
8 #include <comutil.h>
9 #include <oleacc.h>
10 
11 #include <vector>
12 
13 #include "flutter/fml/macros.h"
14 #include "flutter/shell/platform/embedder/embedder.h"
15 #include "flutter/shell/platform/embedder/test_utils/proc_table_replacement.h"
16 #include "flutter/shell/platform/windows/flutter_platform_node_delegate_windows.h"
17 #include "flutter/shell/platform/windows/flutter_windows_engine.h"
18 #include "flutter/shell/platform/windows/flutter_windows_view.h"
19 #include "flutter/shell/platform/windows/testing/engine_modifier.h"
20 #include "flutter/shell/platform/windows/testing/mock_window_binding_handler.h"
21 #include "flutter/shell/platform/windows/testing/test_keyboard.h"
22 #include "gmock/gmock.h"
23 #include "gtest/gtest.h"
24 
25 namespace flutter {
26 namespace testing {
27 
28 namespace {
29 using ::testing::NiceMock;
30 
31 // A structure representing a Win32 MSAA event targeting a specified node.
32 struct MsaaEvent {
33  std::shared_ptr<FlutterPlatformNodeDelegateWindows> node_delegate;
34  ax::mojom::Event event_type;
35 };
36 
37 // Accessibility bridge delegate that captures events dispatched to the OS.
38 class AccessibilityBridgeWindowsSpy : public AccessibilityBridgeWindows {
39  public:
40  using AccessibilityBridgeWindows::OnAccessibilityEvent;
41 
42  explicit AccessibilityBridgeWindowsSpy(FlutterWindowsEngine* engine,
43  FlutterWindowsView* view)
44  : AccessibilityBridgeWindows(view) {}
45 
46  void DispatchWinAccessibilityEvent(
47  std::shared_ptr<FlutterPlatformNodeDelegateWindows> node_delegate,
48  ax::mojom::Event event_type) override {
49  dispatched_events_.push_back({node_delegate, event_type});
50  }
51 
52  void SetFocus(std::shared_ptr<FlutterPlatformNodeDelegateWindows>
53  node_delegate) override {
54  focused_nodes_.push_back(std::move(node_delegate));
55  }
56 
57  void ResetRecords() {
58  dispatched_events_.clear();
59  focused_nodes_.clear();
60  }
61 
62  const std::vector<MsaaEvent>& dispatched_events() const {
63  return dispatched_events_;
64  }
65 
66  const std::vector<int32_t> focused_nodes() const {
67  std::vector<int32_t> ids;
68  std::transform(focused_nodes_.begin(), focused_nodes_.end(),
69  std::back_inserter(ids),
70  [](std::shared_ptr<FlutterPlatformNodeDelegate> node) {
71  return node->GetAXNode()->id();
72  });
73  return ids;
74  }
75 
76  protected:
77  std::weak_ptr<FlutterPlatformNodeDelegate> GetFocusedNode() override {
78  return focused_nodes_.back();
79  }
80 
81  private:
82  std::vector<MsaaEvent> dispatched_events_;
83  std::vector<std::shared_ptr<FlutterPlatformNodeDelegate>> focused_nodes_;
84 
85  FML_DISALLOW_COPY_AND_ASSIGN(AccessibilityBridgeWindowsSpy);
86 };
87 
88 // A FlutterWindowsView whose accessibility bridge is an
89 // AccessibilityBridgeWindowsSpy.
90 class FlutterWindowsViewSpy : public FlutterWindowsView {
91  public:
92  FlutterWindowsViewSpy(FlutterWindowsEngine* engine,
93  std::unique_ptr<WindowBindingHandler> handler)
94  : FlutterWindowsView(kImplicitViewId, engine, std::move(handler)) {}
95 
96  protected:
97  virtual std::shared_ptr<AccessibilityBridgeWindows>
98  CreateAccessibilityBridge() override {
99  return std::make_shared<AccessibilityBridgeWindowsSpy>(GetEngine(), this);
100  }
101 
102  private:
103  FML_DISALLOW_COPY_AND_ASSIGN(FlutterWindowsViewSpy);
104 };
105 
106 // Returns an engine instance configured with dummy project path values, and
107 // overridden methods for sending platform messages, so that the engine can
108 // respond as if the framework were connected.
109 std::unique_ptr<FlutterWindowsEngine> GetTestEngine() {
110  FlutterDesktopEngineProperties properties = {};
111  properties.assets_path = L"C:\\foo\\flutter_assets";
112  properties.icu_data_path = L"C:\\foo\\icudtl.dat";
113  properties.aot_library_path = L"C:\\foo\\aot.so";
114  FlutterProjectBundle project(properties);
115  auto engine = std::make_unique<FlutterWindowsEngine>(project);
116 
117  EngineModifier modifier(engine.get());
118  modifier.embedder_api().UpdateSemanticsEnabled =
119  [](FLUTTER_API_SYMBOL(FlutterEngine) engine, bool enabled) {
120  return kSuccess;
121  };
122 
123  MockEmbedderApiForKeyboard(modifier,
124  std::make_shared<MockKeyResponseController>());
125 
126  engine->Run();
127  return engine;
128 }
129 
130 // Populates the AXTree associated with the specified bridge with test data.
131 //
132 // node0
133 // / \
134 // node1 node2
135 // / \
136 // node3 node4
137 //
138 // node0 and node2 are grouping nodes. node1 and node2 are static text nodes.
139 // node4 is a static text node with no text, and hence has the "ignored" state.
140 void PopulateAXTree(std::shared_ptr<AccessibilityBridge> bridge) {
141  // Add node 0: root.
142  FlutterSemanticsNode2 node0{sizeof(FlutterSemanticsNode2), 0};
143  std::vector<int32_t> node0_children{1, 2};
144  node0.child_count = node0_children.size();
145  node0.children_in_traversal_order = node0_children.data();
146  node0.children_in_hit_test_order = node0_children.data();
147 
148  // Add node 1: text child of node 0.
149  FlutterSemanticsNode2 node1{sizeof(FlutterSemanticsNode2), 1};
150  node1.label = "prefecture";
151  node1.value = "Kyoto";
152 
153  // Add node 2: subtree child of node 0.
154  FlutterSemanticsNode2 node2{sizeof(FlutterSemanticsNode2), 2};
155  std::vector<int32_t> node2_children{3, 4};
156  node2.child_count = node2_children.size();
157  node2.children_in_traversal_order = node2_children.data();
158  node2.children_in_hit_test_order = node2_children.data();
159 
160  // Add node 3: text child of node 2.
161  FlutterSemanticsNode2 node3{sizeof(FlutterSemanticsNode2), 3};
162  node3.label = "city";
163  node3.value = "Uji";
164 
165  // Add node 4: text child (with no text) of node 2.
166  FlutterSemanticsNode2 node4{sizeof(FlutterSemanticsNode2), 4};
167 
168  bridge->AddFlutterSemanticsNodeUpdate(node0);
169  bridge->AddFlutterSemanticsNodeUpdate(node1);
170  bridge->AddFlutterSemanticsNodeUpdate(node2);
171  bridge->AddFlutterSemanticsNodeUpdate(node3);
172  bridge->AddFlutterSemanticsNodeUpdate(node4);
173  bridge->CommitUpdates();
174 }
175 
176 ui::AXNode* AXNodeFromID(std::shared_ptr<AccessibilityBridge> bridge,
177  int32_t id) {
178  auto node_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(id).lock();
179  return node_delegate ? node_delegate->GetAXNode() : nullptr;
180 }
181 
182 std::shared_ptr<AccessibilityBridgeWindowsSpy> GetAccessibilityBridgeSpy(
183  FlutterWindowsView& view) {
184  return std::static_pointer_cast<AccessibilityBridgeWindowsSpy>(
185  view.accessibility_bridge().lock());
186 }
187 
188 void ExpectWinEventFromAXEvent(int32_t node_id,
189  ui::AXEventGenerator::Event ax_event,
190  ax::mojom::Event expected_event) {
191  auto engine = GetTestEngine();
192  FlutterWindowsViewSpy view{
193  engine.get(), std::make_unique<NiceMock<MockWindowBindingHandler>>()};
194  EngineModifier modifier{engine.get()};
195  modifier.SetImplicitView(&view);
196  view.OnUpdateSemanticsEnabled(true);
197 
198  auto bridge = GetAccessibilityBridgeSpy(view);
199  PopulateAXTree(bridge);
200 
201  bridge->ResetRecords();
202  bridge->OnAccessibilityEvent({AXNodeFromID(bridge, node_id),
203  {ax_event, ax::mojom::EventFrom::kNone, {}}});
204  ASSERT_EQ(bridge->dispatched_events().size(), 1);
205  EXPECT_EQ(bridge->dispatched_events()[0].event_type, expected_event);
206 }
207 
208 void ExpectWinEventFromAXEventOnFocusNode(int32_t node_id,
209  ui::AXEventGenerator::Event ax_event,
210  ax::mojom::Event expected_event,
211  int32_t focus_id) {
212  auto engine = GetTestEngine();
213  FlutterWindowsViewSpy view{
214  engine.get(), std::make_unique<NiceMock<MockWindowBindingHandler>>()};
215  EngineModifier modifier{engine.get()};
216  modifier.SetImplicitView(&view);
217  view.OnUpdateSemanticsEnabled(true);
218 
219  auto bridge = GetAccessibilityBridgeSpy(view);
220  PopulateAXTree(bridge);
221 
222  bridge->ResetRecords();
223  auto focus_delegate =
224  bridge->GetFlutterPlatformNodeDelegateFromID(focus_id).lock();
225  bridge->SetFocus(std::static_pointer_cast<FlutterPlatformNodeDelegateWindows>(
226  focus_delegate));
227  bridge->OnAccessibilityEvent({AXNodeFromID(bridge, node_id),
228  {ax_event, ax::mojom::EventFrom::kNone, {}}});
229  ASSERT_EQ(bridge->dispatched_events().size(), 1);
230  EXPECT_EQ(bridge->dispatched_events()[0].event_type, expected_event);
231  EXPECT_EQ(bridge->dispatched_events()[0].node_delegate->GetAXNode()->id(),
232  focus_id);
233 }
234 
235 } // namespace
236 
237 TEST(AccessibilityBridgeWindows, GetParent) {
238  auto engine = GetTestEngine();
239  FlutterWindowsViewSpy view{
240  engine.get(), std::make_unique<NiceMock<MockWindowBindingHandler>>()};
241  EngineModifier modifier{engine.get()};
242  modifier.SetImplicitView(&view);
243  view.OnUpdateSemanticsEnabled(true);
244 
245  auto bridge = view.accessibility_bridge().lock();
246  PopulateAXTree(bridge);
247 
248  auto node0_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(0).lock();
249  auto node1_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(1).lock();
250  EXPECT_EQ(node0_delegate->GetNativeViewAccessible(),
251  node1_delegate->GetParent());
252 }
253 
254 TEST(AccessibilityBridgeWindows, GetParentOnRootRetunsNullptr) {
255  auto engine = GetTestEngine();
256  FlutterWindowsViewSpy view{
257  engine.get(), std::make_unique<NiceMock<MockWindowBindingHandler>>()};
258  EngineModifier modifier{engine.get()};
259  modifier.SetImplicitView(&view);
260  view.OnUpdateSemanticsEnabled(true);
261 
262  auto bridge = view.accessibility_bridge().lock();
263  PopulateAXTree(bridge);
264 
265  auto node0_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(0).lock();
266  ASSERT_TRUE(node0_delegate->GetParent() == nullptr);
267 }
268 
269 TEST(AccessibilityBridgeWindows, DispatchAccessibilityAction) {
270  auto engine = GetTestEngine();
271  FlutterWindowsViewSpy view{
272  engine.get(), std::make_unique<NiceMock<MockWindowBindingHandler>>()};
273  EngineModifier modifier{engine.get()};
274  modifier.SetImplicitView(&view);
275  view.OnUpdateSemanticsEnabled(true);
276 
277  auto bridge = view.accessibility_bridge().lock();
278  PopulateAXTree(bridge);
279 
280  FlutterSemanticsAction actual_action = kFlutterSemanticsActionTap;
281  modifier.embedder_api().DispatchSemanticsAction = MOCK_ENGINE_PROC(
282  DispatchSemanticsAction,
283  ([&actual_action](FLUTTER_API_SYMBOL(FlutterEngine) engine, uint64_t id,
284  FlutterSemanticsAction action, const uint8_t* data,
285  size_t data_length) {
286  actual_action = action;
287  return kSuccess;
288  }));
289 
290  AccessibilityBridgeWindows delegate(&view);
291  delegate.DispatchAccessibilityAction(1, kFlutterSemanticsActionCopy, {});
292  EXPECT_EQ(actual_action, kFlutterSemanticsActionCopy);
293 }
294 
295 TEST(AccessibilityBridgeWindows, OnAccessibilityEventAlert) {
296  ExpectWinEventFromAXEvent(0, ui::AXEventGenerator::Event::ALERT,
297  ax::mojom::Event::kAlert);
298 }
299 
300 TEST(AccessibilityBridgeWindows, OnAccessibilityEventChildrenChanged) {
301  ExpectWinEventFromAXEvent(0, ui::AXEventGenerator::Event::CHILDREN_CHANGED,
302  ax::mojom::Event::kChildrenChanged);
303 }
304 
305 TEST(AccessibilityBridgeWindows, OnAccessibilityEventFocusChanged) {
306  auto engine = GetTestEngine();
307  FlutterWindowsViewSpy view{
308  engine.get(), std::make_unique<NiceMock<MockWindowBindingHandler>>()};
309  EngineModifier modifier{engine.get()};
310  modifier.SetImplicitView(&view);
311  view.OnUpdateSemanticsEnabled(true);
312 
313  auto bridge = GetAccessibilityBridgeSpy(view);
314  PopulateAXTree(bridge);
315 
316  bridge->ResetRecords();
317  bridge->OnAccessibilityEvent({AXNodeFromID(bridge, 1),
318  {ui::AXEventGenerator::Event::FOCUS_CHANGED,
319  ax::mojom::EventFrom::kNone,
320  {}}});
321  ASSERT_EQ(bridge->dispatched_events().size(), 1);
322  EXPECT_EQ(bridge->dispatched_events()[0].event_type,
323  ax::mojom::Event::kFocus);
324 
325  ASSERT_EQ(bridge->focused_nodes().size(), 1);
326  EXPECT_EQ(bridge->focused_nodes()[0], 1);
327 }
328 
329 TEST(AccessibilityBridgeWindows, OnAccessibilityEventIgnoredChanged) {
330  // Static test nodes with no text, hint, or scrollability are ignored.
331  ExpectWinEventFromAXEvent(4, ui::AXEventGenerator::Event::IGNORED_CHANGED,
332  ax::mojom::Event::kHide);
333 }
334 
335 TEST(AccessibilityBridgeWindows, OnAccessibilityImageAnnotationChanged) {
336  ExpectWinEventFromAXEvent(
337  1, ui::AXEventGenerator::Event::IMAGE_ANNOTATION_CHANGED,
338  ax::mojom::Event::kTextChanged);
339 }
340 
341 TEST(AccessibilityBridgeWindows, OnAccessibilityLiveRegionChanged) {
342  ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::LIVE_REGION_CHANGED,
343  ax::mojom::Event::kLiveRegionChanged);
344 }
345 
346 TEST(AccessibilityBridgeWindows, OnAccessibilityNameChanged) {
347  ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::NAME_CHANGED,
348  ax::mojom::Event::kTextChanged);
349 }
350 
351 TEST(AccessibilityBridgeWindows, OnAccessibilityHScrollPosChanged) {
352  ExpectWinEventFromAXEvent(
353  1, ui::AXEventGenerator::Event::SCROLL_HORIZONTAL_POSITION_CHANGED,
354  ax::mojom::Event::kScrollPositionChanged);
355 }
356 
357 TEST(AccessibilityBridgeWindows, OnAccessibilityVScrollPosChanged) {
358  ExpectWinEventFromAXEvent(
359  1, ui::AXEventGenerator::Event::SCROLL_VERTICAL_POSITION_CHANGED,
360  ax::mojom::Event::kScrollPositionChanged);
361 }
362 
363 TEST(AccessibilityBridgeWindows, OnAccessibilitySelectedChanged) {
364  ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::SELECTED_CHANGED,
365  ax::mojom::Event::kValueChanged);
366 }
367 
368 TEST(AccessibilityBridgeWindows, OnAccessibilitySelectedChildrenChanged) {
369  ExpectWinEventFromAXEvent(
370  2, ui::AXEventGenerator::Event::SELECTED_CHILDREN_CHANGED,
371  ax::mojom::Event::kSelectedChildrenChanged);
372 }
373 
374 TEST(AccessibilityBridgeWindows, OnAccessibilitySubtreeCreated) {
375  ExpectWinEventFromAXEvent(0, ui::AXEventGenerator::Event::SUBTREE_CREATED,
376  ax::mojom::Event::kShow);
377 }
378 
379 TEST(AccessibilityBridgeWindows, OnAccessibilityValueChanged) {
380  ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::VALUE_CHANGED,
381  ax::mojom::Event::kValueChanged);
382 }
383 
384 TEST(AccessibilityBridgeWindows, OnAccessibilityStateChanged) {
385  ExpectWinEventFromAXEvent(
386  1, ui::AXEventGenerator::Event::WIN_IACCESSIBLE_STATE_CHANGED,
387  ax::mojom::Event::kStateChanged);
388 }
389 
390 TEST(AccessibilityBridgeWindows, OnDocumentSelectionChanged) {
391  ExpectWinEventFromAXEventOnFocusNode(
392  1, ui::AXEventGenerator::Event::DOCUMENT_SELECTION_CHANGED,
393  ax::mojom::Event::kDocumentSelectionChanged, 2);
394 }
395 
396 } // namespace testing
397 } // namespace flutter
flutter::kImplicitViewId
constexpr FlutterViewId kImplicitViewId
Definition: flutter_windows_engine.h:55
FlutterDesktopEngineProperties::aot_library_path
const wchar_t * aot_library_path
Definition: flutter_windows.h:54
FlutterDesktopEngineProperties
Definition: flutter_windows.h:39
flutter::FlutterEngine
Definition: flutter_engine.h:28
flutter::AccessibilityBridgeWindows::OnAccessibilityEvent
void OnAccessibilityEvent(ui::AXEventGenerator::TargetedEvent targeted_event) override
Handle accessibility events generated due to accessibility tree changes. These events are needed to b...
Definition: accessibility_bridge_windows.cc:18
flutter::AccessibilityBridgeWindows
Definition: accessibility_bridge_windows.h:27
FlutterDesktopEngineProperties::icu_data_path
const wchar_t * icu_data_path
Definition: flutter_windows.h:48
flutter_windows_view.h
accessibility_bridge_windows.h
flutter::AccessibilityBridgeWindows::DispatchAccessibilityAction
void DispatchAccessibilityAction(AccessibilityNodeId target, FlutterSemanticsAction action, fml::MallocMapping data) override
Dispatch accessibility action back to the Flutter framework. These actions are generated in the nativ...
Definition: accessibility_bridge_windows.cc:162
flutter
Definition: accessibility_bridge_windows.cc:11
flutter_windows_engine.h
flutter_platform_node_delegate_windows.h
flutter::testing::TEST
TEST(AccessibilityBridgeWindows, GetParent)
Definition: accessibility_bridge_windows_unittests.cc:237
action
int action
Definition: keyboard_key_handler_unittests.cc:116
event_type
ax::mojom::Event event_type
Definition: accessibility_bridge_windows_unittests.cc:34
FlutterDesktopEngineProperties::assets_path
const wchar_t * assets_path
Definition: flutter_windows.h:43
node_delegate
std::shared_ptr< FlutterPlatformNodeDelegateWindows > node_delegate
Definition: accessibility_bridge_windows_unittests.cc:33