diff --git a/engine/src/flutter/shell/platform/fuchsia/flutter/accessibility_bridge.cc b/engine/src/flutter/shell/platform/fuchsia/flutter/accessibility_bridge.cc index b28d5811af2..1494307325e 100644 --- a/engine/src/flutter/shell/platform/fuchsia/flutter/accessibility_bridge.cc +++ b/engine/src/flutter/shell/platform/fuchsia/flutter/accessibility_bridge.cc @@ -4,6 +4,8 @@ #include "flutter/shell/platform/fuchsia/flutter/accessibility_bridge.h" +#include +#include #include #include @@ -11,13 +13,240 @@ #include "flutter/fml/logging.h" #include "flutter/lib/ui/semantics/semantics_node.h" +#include "runtime/dart/utils/root_inspect_node.h" namespace flutter_runner { +namespace { + +// Returns the ViewRef's koid. +zx_koid_t GetKoid(const fuchsia::ui::views::ViewRef& view_ref) { + zx_handle_t handle = view_ref.reference.get(); + zx_info_handle_basic_t info; + zx_status_t status = zx_object_get_info(handle, ZX_INFO_HANDLE_BASIC, &info, + sizeof(info), nullptr, nullptr); + return status == ZX_OK ? info.koid : ZX_KOID_INVALID; +} + +#if !FLUTTER_RELEASE +static constexpr char kTreeDumpInspectRootName[] = "semantic_tree_root"; + +// Converts flutter semantic node flags to a string representation. +std::string NodeFlagsToString(const flutter::SemanticsNode& node) { + std::string output; + if (node.HasFlag(flutter::SemanticsFlags::kHasCheckedState)) { + output += "kHasCheckedState|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kHasEnabledState)) { + output += "kHasEnabledState|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kHasImplicitScrolling)) { + output += "kHasImplicitScrolling|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kHasToggledState)) { + output += "kHasToggledState|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kIsButton)) { + output += "kIsButton|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kIsChecked)) { + output += "kIsChecked|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kIsEnabled)) { + output += "kIsEnabled|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kIsFocusable)) { + output += "kIsFocusable|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kIsFocused)) { + output += "kIsFocused|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kIsHeader)) { + output += "kIsHeader|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kIsHidden)) { + output += "kIsHidden|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kIsImage)) { + output += "kIsImage|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kIsInMutuallyExclusiveGroup)) { + output += "kIsInMutuallyExclusiveGroup|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kIsKeyboardKey)) { + output += "kIsKeyboardKey|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kIsLink)) { + output += "kIsLink|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kIsLiveRegion)) { + output += "kIsLiveRegion|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kIsObscured)) { + output += "kIsObscured|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kIsReadOnly)) { + output += "kIsReadOnly|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kIsSelected)) { + output += "kIsSelected|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kIsSlider)) { + output += "kIsSlider|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kIsTextField)) { + output += "kIsTextField|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kIsToggled)) { + output += "kIsToggled|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kNamesRoute)) { + output += "kNamesRoute|"; + } + if (node.HasFlag(flutter::SemanticsFlags::kScopesRoute)) { + output += "kScopesRoute|"; + } + + return output; +} + +// Converts flutter semantic node actions to a string representation. +std::string NodeActionsToString(const flutter::SemanticsNode& node) { + std::string output; + + if (node.HasAction(flutter::SemanticsAction::kCopy)) { + output += "kCopy|"; + } + if (node.HasAction(flutter::SemanticsAction::kCustomAction)) { + output += "kCustomAction|"; + } + if (node.HasAction(flutter::SemanticsAction::kCut)) { + output += "kCut|"; + } + if (node.HasAction(flutter::SemanticsAction::kDecrease)) { + output += "kDecrease|"; + } + if (node.HasAction(flutter::SemanticsAction::kDidGainAccessibilityFocus)) { + output += "kDidGainAccessibilityFocus|"; + } + if (node.HasAction(flutter::SemanticsAction::kDidLoseAccessibilityFocus)) { + output += "kDidLoseAccessibilityFocus|"; + } + if (node.HasAction(flutter::SemanticsAction::kDismiss)) { + output += "kDismiss|"; + } + if (node.HasAction(flutter::SemanticsAction::kIncrease)) { + output += "kIncrease|"; + } + if (node.HasAction(flutter::SemanticsAction::kLongPress)) { + output += "kLongPress|"; + } + if (node.HasAction( + flutter::SemanticsAction::kMoveCursorBackwardByCharacter)) { + output += "kMoveCursorBackwardByCharacter|"; + } + if (node.HasAction( + flutter::SemanticsAction::kMoveCursorBackwardByWordIndex)) { + output += "kMoveCursorBackwardByWordIndex|"; + } + if (node.HasAction(flutter::SemanticsAction::kMoveCursorForwardByCharacter)) { + output += "kMoveCursorForwardByCharacter|"; + } + if (node.HasAction(flutter::SemanticsAction::kMoveCursorForwardByWordIndex)) { + output += "kMoveCursorForwardByWordIndex|"; + } + if (node.HasAction(flutter::SemanticsAction::kPaste)) { + output += "kPaste|"; + } + if (node.HasAction(flutter::SemanticsAction::kScrollDown)) { + output += "kScrollDown|"; + } + if (node.HasAction(flutter::SemanticsAction::kScrollLeft)) { + output += "kScrollLeft|"; + } + if (node.HasAction(flutter::SemanticsAction::kScrollRight)) { + output += "kScrollRight|"; + } + if (node.HasAction(flutter::SemanticsAction::kScrollUp)) { + output += "kScrollUp|"; + } + if (node.HasAction(flutter::SemanticsAction::kSetSelection)) { + output += "kSetSelection|"; + } + if (node.HasAction(flutter::SemanticsAction::kSetText)) { + output += "kSetText|"; + } + if (node.HasAction(flutter::SemanticsAction::kShowOnScreen)) { + output += "kShowOnScreen|"; + } + if (node.HasAction(flutter::SemanticsAction::kTap)) { + output += "kTap|"; + } + + return output; +} + +// Returns a string representation of the flutter semantic node absolut +// location. +std::string NodeLocationToString(const SkRect& rect) { + auto min_x = rect.fLeft; + auto min_y = rect.fTop; + auto max_x = rect.fRight; + auto max_y = rect.fBottom; + std::string location = + "min(" + std::to_string(min_x) + ", " + std::to_string(min_y) + ") max(" + + std::to_string(max_x) + ", " + std::to_string(max_y) + ")"; + + return location; +} + +// Returns a string representation of the node's different types of children. +std::string NodeChildrenToString(const flutter::SemanticsNode& node) { + std::stringstream output; + if (!node.childrenInTraversalOrder.empty()) { + output << "children in traversal order:["; + for (const auto child_id : node.childrenInTraversalOrder) { + output << child_id << ", "; + } + output << "]\n"; + } + if (!node.childrenInHitTestOrder.empty()) { + output << "children in hit test order:["; + for (const auto child_id : node.childrenInHitTestOrder) { + output << child_id << ", "; + } + output << ']'; + } + + return output.str(); +} +#endif // !FLUTTER_RELEASE + +} // namespace + AccessibilityBridge::AccessibilityBridge( - Delegate& delegate, + SetSemanticsEnabledCallback set_semantics_enabled_callback, + DispatchSemanticsActionCallback dispatch_semantics_action_callback, const std::shared_ptr services, fuchsia::ui::views::ViewRef view_ref) - : delegate_(delegate), binding_(this) { + : AccessibilityBridge(std::move(set_semantics_enabled_callback), + std::move(dispatch_semantics_action_callback), + services, + dart_utils::RootInspectNode::CreateRootChild( + std::to_string(GetKoid(view_ref))), + std::move(view_ref)) {} + +AccessibilityBridge::AccessibilityBridge( + SetSemanticsEnabledCallback set_semantics_enabled_callback, + DispatchSemanticsActionCallback dispatch_semantics_action_callback, + const std::shared_ptr services, + inspect::Node inspect_node, + fuchsia::ui::views::ViewRef view_ref) + : set_semantics_enabled_callback_( + std::move(set_semantics_enabled_callback)), + dispatch_semantics_action_callback_( + std::move(dispatch_semantics_action_callback)), + binding_(this), + inspect_node_(std::move(inspect_node)) { services->Connect(fuchsia::accessibility::semantics::SemanticsManager::Name_, fuchsia_semantics_manager_.NewRequest().TakeChannel()); fuchsia_semantics_manager_.set_error_handler([](zx_status_t status) { @@ -27,6 +256,27 @@ AccessibilityBridge::AccessibilityBridge( fidl::InterfaceHandle listener_handle; binding_.Bind(listener_handle.NewRequest()); + +#if !FLUTTER_RELEASE + // The first argument to |CreateLazyValues| is the name of the lazy node, and + // will only be displayed if the callback used to generate the node's content + // fails. Therefore, we use an error message for this node name. + inspect_node_tree_dump_ = + inspect_node_.CreateLazyValues("dump_fail", [this]() { + inspect::Inspector inspector; + if (auto it = nodes_.find(kRootNodeId); it == nodes_.end()) { + inspector.GetRoot().CreateString( + "empty_tree", "this semantic tree is empty", &inspector); + } else { + FillInspectTree( + kRootNodeId, /*current_level=*/1, + inspector.GetRoot().CreateChild(kTreeDumpInspectRootName), + &inspector); + } + return fit::make_ok_promise(std::move(inspector)); + }); +#endif // !FLUTTER_RELEASE + fuchsia_semantics_manager_->RegisterViewForSemantics( std::move(view_ref), std::move(listener_handle), tree_ptr_.NewRequest()); } @@ -224,14 +474,16 @@ std::unordered_set AccessibilityBridge::GetDescendants( auto it = nodes_.find(id); if (it != nodes_.end()) { - const auto& node = it->second; - for (const auto& child : node.children_in_hit_test_order) { + const auto& node = it->second.data; + for (const auto& child : node.childrenInHitTestOrder) { if (descendents.find(child) == descendents.end()) { to_process.push_back(child); } else { // This indicates either a cycle or a child with multiple parents. // Flutter should never let this happen, but the engine API does not // explicitly forbid it right now. + // TODO(http://fxbug.dev/75905): Crash flutter accessibility bridge + // when a cycle in the tree is found. FML_LOG(ERROR) << "Semantics Node " << child << " has already been listed as a child of another " "node, ignoring for parent " @@ -295,7 +547,7 @@ void AccessibilityBridge::AddSemanticsNodeUpdate( << "AccessibilityBridge received an update with out ever getting a root " "node."; - std::vector nodes; + std::vector fuchsia_nodes; size_t current_size = 0; bool has_root_node_update = false; // TODO(MI4-2498): Actions, Roles, hit test children, additional @@ -303,26 +555,22 @@ void AccessibilityBridge::AddSemanticsNodeUpdate( // TODO(MI4-1478): Support for partial updates for nodes > 64kb // e.g. if a node has a long label or more than 64k children. - for (const auto& value : update) { + for (const auto& [flutter_node_id, flutter_node] : update) { size_t this_node_size = sizeof(fuchsia::accessibility::semantics::Node); - const auto& flutter_node = value.second; // We handle root update separately in GetRootNodeUpdate. // TODO(chunhtai): remove this special case after we remove the inverse // view pixel ratio transformation in scenic view. + // TODO(http://fxbug.dev/75908): Investigate flutter a11y bridge refactor + // after removal of the inverse view pixel ratio transformation in scenic + // view). if (flutter_node.id == kRootNodeId) { root_flutter_semantics_node_ = flutter_node; has_root_node_update = true; continue; } - // Store the nodes for later hit testing. - nodes_[flutter_node.id] = { - .id = flutter_node.id, - .flags = flutter_node.flags, - .is_focusable = IsFocusable(flutter_node), - .rect = flutter_node.rect, - .transform = flutter_node.transform, - .children_in_hit_test_order = flutter_node.childrenInHitTestOrder, - }; + // Store the nodes for later hit testing and logging. + nodes_[flutter_node.id].data = flutter_node; + fuchsia::accessibility::semantics::Node fuchsia_node; std::vector child_ids; // Send the nodes in traversal order, so the manager can figure out @@ -330,6 +578,8 @@ void AccessibilityBridge::AddSemanticsNodeUpdate( for (int32_t flutter_child_id : flutter_node.childrenInTraversalOrder) { child_ids.push_back(FlutterIdToFuchsiaId(flutter_child_id)); } + // TODO(http://fxbug.dev/75910): check the usage of FlutterIdToFuchsiaId in + // the flutter accessibility bridge. fuchsia_node.set_node_id(flutter_node.id) .set_role(GetNodeRole(flutter_node)) .set_location(GetNodeLocation(flutter_node)) @@ -337,7 +587,6 @@ void AccessibilityBridge::AddSemanticsNodeUpdate( .set_attributes(GetNodeAttributes(flutter_node, &this_node_size)) .set_states(GetNodeStates(flutter_node, &this_node_size)) .set_actions(GetNodeActions(flutter_node, &this_node_size)) - .set_role(GetNodeRole(flutter_node)) .set_child_ids(child_ids); this_node_size += kNodeIdSize * flutter_node.childrenInTraversalOrder.size(); @@ -354,15 +603,15 @@ void AccessibilityBridge::AddSemanticsNodeUpdate( // If we would exceed the max FIDL message size by appending this node, // we should delete/update/commit now. if (current_size >= kMaxMessageSize) { - tree_ptr_->UpdateSemanticNodes(std::move(nodes)); - nodes.clear(); + tree_ptr_->UpdateSemanticNodes(std::move(fuchsia_nodes)); + fuchsia_nodes.clear(); current_size = this_node_size; } - nodes.push_back(std::move(fuchsia_node)); + fuchsia_nodes.push_back(std::move(fuchsia_node)); } if (current_size > kMaxMessageSize) { - PrintNodeSizeError(nodes.back().node_id()); + PrintNodeSizeError(fuchsia_nodes.back().node_id()); } // Handles root node update. @@ -382,16 +631,16 @@ void AccessibilityBridge::AddSemanticsNodeUpdate( // If we would exceed the max FIDL message size by appending this node, // we should delete/update/commit now. if (current_size >= kMaxMessageSize) { - tree_ptr_->UpdateSemanticNodes(std::move(nodes)); - nodes.clear(); + tree_ptr_->UpdateSemanticNodes(std::move(fuchsia_nodes)); + fuchsia_nodes.clear(); } - nodes.push_back(std::move(root_update)); + fuchsia_nodes.push_back(std::move(root_update)); } PruneUnreachableNodes(); UpdateScreenRects(); - tree_ptr_->UpdateSemanticNodes(std::move(nodes)); + tree_ptr_->UpdateSemanticNodes(std::move(fuchsia_nodes)); // TODO(dnfield): Implement the callback here // https://bugs.fuchsia.dev/p/fuchsia/issues/detail?id=35718. tree_ptr_->CommitUpdates([]() {}); @@ -414,15 +663,10 @@ fuchsia::accessibility::semantics::Node AccessibilityBridge::GetRootNodeUpdate( SkM44 result = root_flutter_semantics_node_.transform * inverse_view_pixel_ratio_transform; - nodes_[root_flutter_semantics_node_.id] = { - .id = root_flutter_semantics_node_.id, - .flags = root_flutter_semantics_node_.flags, - .is_focusable = IsFocusable(root_flutter_semantics_node_), - .rect = root_flutter_semantics_node_.rect, - .transform = result, - .children_in_hit_test_order = - root_flutter_semantics_node_.childrenInHitTestOrder, - }; + nodes_[root_flutter_semantics_node_.id].data = root_flutter_semantics_node_; + + // TODO(http://fxbug.dev/75910): check the usage of FlutterIdToFuchsiaId in + // the flutter accessibility bridge. root_fuchsia_node.set_node_id(root_flutter_semantics_node_.id) .set_role(GetNodeRole(root_flutter_semantics_node_)) .set_location(GetNodeLocation(root_flutter_semantics_node_)) @@ -461,9 +705,9 @@ void AccessibilityBridge::UpdateScreenRects( return; } auto& node = it->second; - const auto& current_transform = parent_transform * node.transform; + const auto& current_transform = parent_transform * node.data.transform; - const auto& rect = node.rect; + const auto& rect = node.data.rect; SkV4 dst[2] = { current_transform.map(rect.left(), rect.top(), 0, 1), current_transform.map(rect.right(), rect.bottom(), 0, 1), @@ -473,7 +717,7 @@ void AccessibilityBridge::UpdateScreenRects( visited_nodes->emplace(node_id); - for (uint32_t child_id : node.children_in_hit_test_order) { + for (uint32_t child_id : node.data.childrenInHitTestOrder) { if (visited_nodes->find(child_id) == visited_nodes->end()) { UpdateScreenRects(child_id, current_transform, visited_nodes); } @@ -525,6 +769,8 @@ void AccessibilityBridge::OnAccessibilityActionRequested( fuchsia::accessibility::semantics::Action action, fuchsia::accessibility::semantics::SemanticListener:: OnAccessibilityActionRequestedCallback callback) { + // TODO(http://fxbug.dev/75910): check the usage of FlutterIdToFuchsiaId in + // the flutter accessibility bridge. if (nodes_.find(node_id) == nodes_.end()) { FML_LOG(ERROR) << "Attempted to send accessibility action " << static_cast(action) @@ -539,8 +785,8 @@ void AccessibilityBridge::OnAccessibilityActionRequested( callback(false); return; } - delegate_.DispatchSemanticsAction(static_cast(node_id), - flutter_action.value()); + dispatch_semantics_action_callback_(static_cast(node_id), + flutter_action.value()); callback(true); } @@ -552,6 +798,8 @@ void AccessibilityBridge::HitTest( auto hit_node_id = GetHitNode(kRootNodeId, local_point.x, local_point.y); FML_DCHECK(hit_node_id.has_value()); fuchsia::accessibility::semantics::Hit hit; + // TODO(http://fxbug.dev/75910): check the usage of FlutterIdToFuchsiaId in + // the flutter accessibility bridge. hit.set_node_id(hit_node_id.value_or(kRootNodeId)); callback(std::move(hit)); } @@ -565,19 +813,19 @@ std::optional AccessibilityBridge::GetHitNode(int32_t node_id, return {}; } auto const& node = it->second; - if (node.flags & + if (node.data.flags & static_cast(flutter::SemanticsFlags::kIsHidden) || // !node.screen_rect.contains(x, y)) { return {}; } - for (int32_t child_id : node.children_in_hit_test_order) { + for (int32_t child_id : node.data.childrenInHitTestOrder) { auto candidate = GetHitNode(child_id, x, y); if (candidate) { return candidate; } } - if (node.is_focusable) { + if (IsFocusable(node.data)) { return node_id; } @@ -599,7 +847,7 @@ bool AccessibilityBridge::IsFocusable( return true; } - // Always conider actionable nodes focusable. + // Always consider actionable nodes focusable. if (node.actions != 0) { return true; } @@ -612,7 +860,76 @@ bool AccessibilityBridge::IsFocusable( void AccessibilityBridge::OnSemanticsModeChanged( bool enabled, OnSemanticsModeChangedCallback callback) { - delegate_.SetSemanticsEnabled(enabled); + set_semantics_enabled_callback_(enabled); } +#if !FLUTTER_RELEASE +void AccessibilityBridge::FillInspectTree(int32_t flutter_node_id, + int32_t current_level, + inspect::Node inspect_node, + inspect::Inspector* inspector) const { + const auto it = nodes_.find(flutter_node_id); + if (it == nodes_.end()) { + inspect_node.CreateString( + "missing_child", + "This node has a parent in the semantic tree but has no value", + inspector); + inspector->emplace(std::move(inspect_node)); + return; + } + const auto& semantic_node = it->second; + const auto& data = semantic_node.data; + + inspect_node.CreateInt("id", data.id, inspector); + + // Even with an empty label, we still want to create the property to + // explicetly show that it is empty. + inspect_node.CreateString("label", data.label, inspector); + if (!data.hint.empty()) { + inspect_node.CreateString("hint", data.hint, inspector); + } + if (!data.value.empty()) { + inspect_node.CreateString("value", data.value, inspector); + } + if (!data.increasedValue.empty()) { + inspect_node.CreateString("increased_value", data.increasedValue, + inspector); + } + if (!data.decreasedValue.empty()) { + inspect_node.CreateString("decreased_value", data.decreasedValue, + inspector); + } + + if (data.textDirection) { + inspect_node.CreateString( + "text_direction", data.textDirection == 1 ? "RTL" : "LTR", inspector); + } + + if (data.flags) { + inspect_node.CreateString("flags", NodeFlagsToString(data), inspector); + } + if (data.actions) { + inspect_node.CreateString("actions", NodeActionsToString(data), inspector); + } + + inspect_node.CreateString( + "location", NodeLocationToString(semantic_node.screen_rect), inspector); + if (!data.childrenInTraversalOrder.empty() || + !data.childrenInHitTestOrder.empty()) { + inspect_node.CreateString("children", NodeChildrenToString(data), + inspector); + } + + inspect_node.CreateInt("current_level", current_level, inspector); + + for (int32_t flutter_child_id : semantic_node.data.childrenInTraversalOrder) { + const auto inspect_name = "node_" + std::to_string(flutter_child_id); + FillInspectTree(flutter_child_id, current_level + 1, + inspect_node.CreateChild(inspect_name), inspector); + } + + inspector->emplace(std::move(inspect_node)); +} +#endif // !FLUTTER_RELEASE + } // namespace flutter_runner diff --git a/engine/src/flutter/shell/platform/fuchsia/flutter/accessibility_bridge.h b/engine/src/flutter/shell/platform/fuchsia/flutter/accessibility_bridge.h index 8b0e9390471..b0c34b0dc40 100644 --- a/engine/src/flutter/shell/platform/fuchsia/flutter/accessibility_bridge.h +++ b/engine/src/flutter/shell/platform/fuchsia/flutter/accessibility_bridge.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -40,13 +41,9 @@ namespace flutter_runner { class AccessibilityBridge : public fuchsia::accessibility::semantics::SemanticListener { public: - // A delegate to call when semantics are enabled or disabled. - class Delegate { - public: - virtual void SetSemanticsEnabled(bool enabled) = 0; - virtual void DispatchSemanticsAction(int32_t node_id, - flutter::SemanticsAction action) = 0; - }; + using SetSemanticsEnabledCallback = std::function; + using DispatchSemanticsActionCallback = + std::function; // TODO(MI4-2531, FIDL-718): Remove this. We shouldn't be worried about // batching messages at this level. @@ -67,9 +64,18 @@ class AccessibilityBridge "flutter::SemanticsNode::id and " "fuchsia::accessibility::semantics::Node::node_id differ in size."); - AccessibilityBridge(Delegate& delegate, - const std::shared_ptr services, - fuchsia::ui::views::ViewRef view_ref); + AccessibilityBridge( + SetSemanticsEnabledCallback set_semantics_enabled_callback, + DispatchSemanticsActionCallback dispatch_semantics_action_callback, + const std::shared_ptr services, + fuchsia::ui::views::ViewRef view_ref); + + AccessibilityBridge( + SetSemanticsEnabledCallback set_semantics_enabled_callback, + DispatchSemanticsActionCallback dispatch_semantics_action_callback, + const std::shared_ptr services, + inspect::Node inspect_node, + fuchsia::ui::views::ViewRef view_ref); // Returns true if accessible navigation is enabled. bool GetSemanticsEnabled() const; @@ -105,28 +111,16 @@ class AccessibilityBridge OnAccessibilityActionRequestedCallback callback) override; private: - // Holds only the fields we need for hit testing. + // Fuchsia's default root semantic node id. + static constexpr int32_t kRootNodeId = 0; + + // Holds a flutter semantic node and some extra info. // In particular, it adds a screen_rect field to flutter::SemanticsNode. struct SemanticsNode { - int32_t id; - int32_t flags; - bool is_focusable; - SkRect rect; + flutter::SemanticsNode data; SkRect screen_rect; - SkM44 transform; - std::vector children_in_hit_test_order; }; - AccessibilityBridge::Delegate& delegate_; - - static constexpr int32_t kRootNodeId = 0; - flutter::SemanticsNode root_flutter_semantics_node_; - float last_seen_view_pixel_ratio_ = 1.f; - fidl::Binding binding_; - fuchsia::accessibility::semantics::SemanticsManagerPtr - fuchsia_semantics_manager_; - fuchsia::accessibility::semantics::SemanticTreePtr tree_ptr_; - bool semantics_enabled_; // This is the cache of all nodes we've sent to Fuchsia's SemanticsManager. // Assists with pruning unreachable nodes and hit testing. std::unordered_map nodes_; @@ -214,6 +208,34 @@ class AccessibilityBridge void OnSemanticsModeChanged(bool enabled, OnSemanticsModeChangedCallback callback) override; +#if !FLUTTER_RELEASE + // Fills the inspect tree with debug information about the semantic tree. + void FillInspectTree(int32_t flutter_node_id, + int32_t current_level, + inspect::Node inspect_node, + inspect::Inspector* inspector) const; +#endif // !FLUTTER_RELEASE + + SetSemanticsEnabledCallback set_semantics_enabled_callback_; + DispatchSemanticsActionCallback dispatch_semantics_action_callback_; + flutter::SemanticsNode root_flutter_semantics_node_; + float last_seen_view_pixel_ratio_ = 1.f; + fidl::Binding binding_; + fuchsia::accessibility::semantics::SemanticsManagerPtr + fuchsia_semantics_manager_; + fuchsia::accessibility::semantics::SemanticTreePtr tree_ptr_; + bool semantics_enabled_; + + // Node to publish inspect data. + inspect::Node inspect_node_; + +#if !FLUTTER_RELEASE + // Inspect node to store a dump of the semantic tree. Note that this only gets + // computed if requested, so it does not use memory to store the dump unless + // an explicit request is made. + inspect::LazyNode inspect_node_tree_dump_; +#endif // !FLUTTER_RELEASE + FML_DISALLOW_COPY_AND_ASSIGN(AccessibilityBridge); }; } // namespace flutter_runner diff --git a/engine/src/flutter/shell/platform/fuchsia/flutter/accessibility_bridge_unittest.cc b/engine/src/flutter/shell/platform/fuchsia/flutter/accessibility_bridge_unittest.cc index cd171b113de..e61e08ecebf 100644 --- a/engine/src/flutter/shell/platform/fuchsia/flutter/accessibility_bridge_unittest.cc +++ b/engine/src/flutter/shell/platform/fuchsia/flutter/accessibility_bridge_unittest.cc @@ -7,8 +7,12 @@ #include #include #include +#include #include #include +#include +#include +#include #include #include @@ -33,12 +37,11 @@ void ExpectNodeHasRole( } // namespace -class AccessibilityBridgeTestDelegate - : public flutter_runner::AccessibilityBridge::Delegate { +class AccessibilityBridgeTestDelegate { public: - void SetSemanticsEnabled(bool enabled) override { enabled_ = enabled; } + void SetSemanticsEnabled(bool enabled) { enabled_ = enabled; } void DispatchSemanticsAction(int32_t node_id, - flutter::SemanticsAction action) override { + flutter::SemanticsAction action) { actions.push_back(std::make_pair(node_id, action)); } @@ -53,7 +56,8 @@ class AccessibilityBridgeTest : public testing::Test { public: AccessibilityBridgeTest() : loop_(&kAsyncLoopConfigAttachToCurrentThread), - services_provider_(loop_.dispatcher()) { + services_provider_(loop_.dispatcher()), + executor_(loop_.dispatcher()) { services_provider_.AddService( semantics_manager_.GetHandler(loop_.dispatcher()), SemanticsManager::Name_); @@ -64,6 +68,21 @@ class AccessibilityBridgeTest : public testing::Test { loop_.ResetQuit(); } + void RunPromiseToCompletion(fit::promise<> promise) { + bool done = false; + executor_.schedule_task( + std::move(promise).and_then([&done]() { done = true; })); + while (loop_.GetState() == ASYNC_LOOP_RUNNABLE) { + if (done) { + loop_.ResetQuit(); + return; + } + + loop_.Run(zx::deadline_after(zx::duration::infinite()), true); + } + loop_.ResetQuit(); + } + protected: void SetUp() override { zx_status_t status = zx::eventpair::create( @@ -71,9 +90,22 @@ class AccessibilityBridgeTest : public testing::Test { EXPECT_EQ(status, ZX_OK); accessibility_delegate_.actions.clear(); + inspector_ = std::make_unique(); + flutter_runner::AccessibilityBridge::SetSemanticsEnabledCallback + set_semantics_enabled_callback = [this](bool enabled) { + accessibility_delegate_.SetSemanticsEnabled(enabled); + }; + flutter_runner::AccessibilityBridge::DispatchSemanticsActionCallback + dispatch_semantics_action_callback = + [this](int32_t node_id, flutter::SemanticsAction action) { + accessibility_delegate_.DispatchSemanticsAction(node_id, action); + }; accessibility_bridge_ = std::make_unique( - accessibility_delegate_, services_provider_.service_directory(), + std::move(set_semantics_enabled_callback), + std::move(dispatch_semantics_action_callback), + services_provider_.service_directory(), + inspector_->GetRoot().CreateChild("test_node"), std::move(view_ref_)); RunLoopUntilIdle(); } @@ -85,10 +117,14 @@ class AccessibilityBridgeTest : public testing::Test { MockSemanticsManager semantics_manager_; AccessibilityBridgeTestDelegate accessibility_delegate_; std::unique_ptr accessibility_bridge_; + // Required to verify inspect metrics. + std::unique_ptr inspector_; private: async::Loop loop_; sys::testing::ServiceDirectoryProvider services_provider_; + // Required to retrieve inspect metrics. + async::Executor executor_; }; TEST_F(AccessibilityBridgeTest, RegistersViewRef) { @@ -952,4 +988,54 @@ TEST_F(AccessibilityBridgeTest, Actions) { EXPECT_EQ(accessibility_delegate_.actions.back(), std::make_pair(0, flutter::SemanticsAction::kDecrease)); } + +#if !FLUTTER_RELEASE +TEST_F(AccessibilityBridgeTest, InspectData) { + flutter::SemanticsNodeUpdates updates; + flutter::SemanticsNode node0; + node0.id = 0; + node0.label = "node0"; + node0.hint = "node0_hint"; + node0.value = "value"; + node0.flags |= static_cast(flutter::SemanticsFlags::kIsButton); + node0.childrenInTraversalOrder = {1}; + node0.childrenInHitTestOrder = {1}; + node0.rect.setLTRB(0, 0, 100, 100); + updates.emplace(0, node0); + + flutter::SemanticsNode node1; + node1.id = 1; + node1.flags |= static_cast(flutter::SemanticsFlags::kIsHeader); + node1.childrenInTraversalOrder = {}; + node1.childrenInHitTestOrder = {}; + updates.emplace(1, node1); + + accessibility_bridge_->AddSemanticsNodeUpdate(std::move(updates), 1.f); + RunLoopUntilIdle(); + + fit::result hierarchy; + ASSERT_FALSE(hierarchy.is_ok()); + RunPromiseToCompletion( + inspect::ReadFromInspector(*inspector_) + .then([&hierarchy](fit::result& result) { + hierarchy = std::move(result); + })); + ASSERT_TRUE(hierarchy.is_ok()); + + auto tree_inspect_hierarchy = hierarchy.value().GetByPath({"test_node"}); + ASSERT_NE(tree_inspect_hierarchy, nullptr); + // TODO(http://fxbug.dev/75841): Rewrite flutter engine accessibility bridge + // tests using inspect matchers. The checks bellow verify that the tree was + // built, and that it matches the format of the input tree. This will be + // updated in the future when test matchers are available to verify individual + // property values. + const auto& root = tree_inspect_hierarchy->children(); + ASSERT_EQ(root.size(), 1u); + EXPECT_EQ(root[0].name(), "semantic_tree_root"); + const auto& child = root[0].children(); + ASSERT_EQ(child.size(), 1u); + EXPECT_EQ(child[0].name(), "node_1"); +} +#endif // !FLUTTER_RELEASE + } // namespace flutter_runner_test diff --git a/engine/src/flutter/shell/platform/fuchsia/flutter/engine.cc b/engine/src/flutter/shell/platform/fuchsia/flutter/engine.cc index 020c110ca74..5a33a52bf3f 100644 --- a/engine/src/flutter/shell/platform/fuchsia/flutter/engine.cc +++ b/engine/src/flutter/shell/platform/fuchsia/flutter/engine.cc @@ -102,6 +102,8 @@ Engine::Engine(Delegate& delegate, // refs are not copyable, and multiple consumers need view refs. fuchsia::ui::views::ViewRef platform_view_ref; view_ref_pair.view_ref.Clone(&platform_view_ref); + fuchsia::ui::views::ViewRef accessibility_bridge_view_ref; + view_ref_pair.view_ref.Clone(&accessibility_bridge_view_ref); fuchsia::ui::views::ViewRef isolate_view_ref; view_ref_pair.view_ref.Clone(&isolate_view_ref); // Input3 keyboard listener registration requires a ViewRef as an event @@ -171,6 +173,30 @@ Engine::Engine(Delegate& delegate, environment->GetServices(parent_environment_service_provider.NewRequest()); environment.Unbind(); + AccessibilityBridge::SetSemanticsEnabledCallback + set_semantics_enabled_callback = [this](bool enabled) { + auto platform_view = shell_->GetPlatformView(); + + if (platform_view) { + platform_view->SetSemanticsEnabled(enabled); + } + }; + + AccessibilityBridge::DispatchSemanticsActionCallback + dispatch_semantics_action_callback = + [this](int32_t node_id, flutter::SemanticsAction action) { + auto platform_view = shell_->GetPlatformView(); + + if (platform_view) { + platform_view->DispatchSemanticsAction(node_id, action, {}); + } + }; + + accessibility_bridge_ = std::make_unique( + std::move(set_semantics_enabled_callback), + std::move(dispatch_semantics_action_callback), svc, + std::move(accessibility_bridge_view_ref)); + OnEnableWireframe on_enable_wireframe_callback = std::bind( &Engine::DebugWireframeSettingsChanged, this, std::placeholders::_1); @@ -227,6 +253,16 @@ Engine::Engine(Delegate& delegate, keyboard_svc_->AddListener(std::move(keyboard_view_ref), keyboard_listener.Bind(), [] {}); + OnSemanticsNodeUpdate on_semantics_node_update_callback = + [this](flutter::SemanticsNodeUpdates updates, float pixel_ratio) { + accessibility_bridge_->AddSemanticsNodeUpdate(updates, pixel_ratio); + }; + + OnRequestAnnounce on_request_announce_callback = + [this](const std::string& message) { + accessibility_bridge_->RequestAnnounce(message); + }; + // Setup the callback that will instantiate the platform view. flutter::Shell::CreateCallback on_create_platform_view = fml::MakeCopyable( @@ -244,6 +280,10 @@ Engine::Engine(Delegate& delegate, on_update_view_callback = std::move(on_update_view_callback), on_destroy_view_callback = std::move(on_destroy_view_callback), on_create_surface_callback = std::move(on_create_surface_callback), + on_semantics_node_update_callback = + std::move(on_semantics_node_update_callback), + on_request_announce_callback = + std::move(on_request_announce_callback), external_view_embedder = GetExternalViewEmbedder(), keyboard_listener_request = std::move(keyboard_listener_request), await_vsync_callback = @@ -271,7 +311,9 @@ Engine::Engine(Delegate& delegate, std::move(on_create_view_callback), std::move(on_update_view_callback), std::move(on_destroy_view_callback), - std::move(on_create_surface_callback), external_view_embedder, + std::move(on_create_surface_callback), + std::move(on_semantics_node_update_callback), + std::move(on_request_announce_callback), external_view_embedder, // Callbacks for VsyncWaiter to call into SessionConnection. await_vsync_callback, await_vsync_for_secondary_callback_callback); diff --git a/engine/src/flutter/shell/platform/fuchsia/flutter/engine.h b/engine/src/flutter/shell/platform/fuchsia/flutter/engine.h index f6303274fe0..fd3b3c99888 100644 --- a/engine/src/flutter/shell/platform/fuchsia/flutter/engine.h +++ b/engine/src/flutter/shell/platform/fuchsia/flutter/engine.h @@ -21,6 +21,7 @@ #include "flutter/flow/surface.h" #include "flutter/fml/macros.h" #include "flutter/shell/common/shell.h" +#include "flutter/shell/platform/fuchsia/flutter/accessibility_bridge.h" #include "default_session_connection.h" #include "flutter_runner_product_configuration.h" @@ -83,6 +84,7 @@ class Engine final { std::unique_ptr isolate_configurator_; std::unique_ptr shell_; + std::unique_ptr accessibility_bridge_; fuchsia::intl::PropertyProviderPtr intl_property_provider_; diff --git a/engine/src/flutter/shell/platform/fuchsia/flutter/platform_view.cc b/engine/src/flutter/shell/platform/fuchsia/flutter/platform_view.cc index f5e739f8b30..d2c9c0f2af9 100644 --- a/engine/src/flutter/shell/platform/fuchsia/flutter/platform_view.cc +++ b/engine/src/flutter/shell/platform/fuchsia/flutter/platform_view.cc @@ -72,6 +72,8 @@ PlatformView::PlatformView( OnUpdateView on_update_view_callback, OnDestroyView on_destroy_view_callback, OnCreateSurface on_create_surface_callback, + OnSemanticsNodeUpdate on_semantics_node_update_callback, + OnRequestAnnounce on_request_announce_callback, std::shared_ptr external_view_embedder, AwaitVsyncCallback await_vsync_callback, AwaitVsyncForSecondaryCallbackCallback @@ -88,6 +90,9 @@ PlatformView::PlatformView( on_update_view_callback_(std::move(on_update_view_callback)), on_destroy_view_callback_(std::move(on_destroy_view_callback)), on_create_surface_callback_(std::move(on_create_surface_callback)), + on_semantics_node_update_callback_( + std::move(on_semantics_node_update_callback)), + on_request_announce_callback_(std::move(on_request_announce_callback)), external_view_embedder_(external_view_embedder), ime_client_(this), keyboard_listener_binding_(this, std::move(keyboard_listener_request)), @@ -113,11 +118,6 @@ PlatformView::PlatformView( // Finally! Register the native platform message handlers. RegisterPlatformMessageHandlers(); - - fuchsia::ui::views::ViewRef accessibility_view_ref; - view_ref_.Clone(&accessibility_view_ref); - accessibility_bridge_ = std::make_unique( - *this, runner_services, std::move(accessibility_view_ref)); } PlatformView::~PlatformView() = default; @@ -734,7 +734,6 @@ void PlatformView::HandlePlatformMessage( } // |flutter::PlatformView| -// |flutter_runner::AccessibilityBridge::Delegate| void PlatformView::SetSemanticsEnabled(bool enabled) { flutter::PlatformView::SetSemanticsEnabled(enabled); if (enabled) { @@ -745,13 +744,6 @@ void PlatformView::SetSemanticsEnabled(bool enabled) { } } -// |flutter::PlatformView| -// |flutter_runner::AccessibilityBridge::Delegate| -void PlatformView::DispatchSemanticsAction(int32_t node_id, - flutter::SemanticsAction action) { - flutter::PlatformView::DispatchSemanticsAction(node_id, action, {}); -} - // |flutter::PlatformView| void PlatformView::UpdateSemantics( flutter::SemanticsNodeUpdates update, @@ -759,7 +751,7 @@ void PlatformView::UpdateSemantics( const float pixel_ratio = view_pixel_ratio_.has_value() ? *view_pixel_ratio_ : 0.f; - accessibility_bridge_->AddSemanticsNodeUpdate(update, pixel_ratio); + on_semantics_node_update_callback_(update, pixel_ratio); } // Channel handler for kAccessibilityChannel @@ -782,7 +774,7 @@ void PlatformView::HandleAccessibilityChannelPlatformMessage( std::string text = std::get(data_map.at(flutter::EncodableValue("message"))); - accessibility_bridge_->RequestAnnounce(text); + on_request_announce_callback_(text); } message->response()->CompleteEmpty(); diff --git a/engine/src/flutter/shell/platform/fuchsia/flutter/platform_view.h b/engine/src/flutter/shell/platform/fuchsia/flutter/platform_view.h index 940a60132cb..91ca8f31e43 100644 --- a/engine/src/flutter/shell/platform/fuchsia/flutter/platform_view.h +++ b/engine/src/flutter/shell/platform/fuchsia/flutter/platform_view.h @@ -5,11 +5,13 @@ #ifndef FLUTTER_SHELL_PLATFORM_FUCHSIA_PLATFORM_VIEW_H_ #define FLUTTER_SHELL_PLATFORM_FUCHSIA_PLATFORM_VIEW_H_ +#include #include #include #include #include #include +#include #include #include @@ -25,8 +27,6 @@ #include "flutter/shell/platform/fuchsia/flutter/keyboard.h" #include "flutter/shell/platform/fuchsia/flutter/vsync_waiter.h" -#include "accessibility_bridge.h" - namespace flutter_runner { using OnEnableWireframe = fit::function; @@ -34,6 +34,9 @@ using OnCreateView = fit::function; using OnUpdateView = fit::function; using OnDestroyView = fit::function; using OnCreateSurface = fit::function()>; +using OnSemanticsNodeUpdate = + fit::function; +using OnRequestAnnounce = fit::function; // PlatformView is the per-engine component residing on the platform thread that // is responsible for all platform specific integrations -- particularly @@ -48,7 +51,6 @@ using OnCreateSurface = fit::function()>; // does *not* actually own the Session itself; that is owned by the // FuchsiaExternalViewEmbedder on the raster thread. class PlatformView final : public flutter::PlatformView, - public AccessibilityBridge::Delegate, private fuchsia::ui::scenic::SessionListener, private fuchsia::ui::input3::KeyboardListener, private fuchsia::ui::input::InputMethodEditorClient { @@ -71,6 +73,8 @@ class PlatformView final : public flutter::PlatformView, OnUpdateView on_update_view_callback, OnDestroyView on_destroy_view_callback, OnCreateSurface on_create_surface_callback, + OnSemanticsNodeUpdate on_semantics_node_update_callback, + OnRequestAnnounce on_request_announce_callback, std::shared_ptr view_embedder, AwaitVsyncCallback await_vsync_callback, AwaitVsyncForSecondaryCallbackCallback @@ -79,13 +83,8 @@ class PlatformView final : public flutter::PlatformView, ~PlatformView(); // |flutter::PlatformView| - // |flutter_runner::AccessibilityBridge::Delegate| void SetSemanticsEnabled(bool enabled) override; - // |flutter_runner::AccessibilityBridge::Delegate| - void DispatchSemanticsAction(int32_t node_id, - flutter::SemanticsAction action) override; - // |flutter::PlatformView| std::shared_ptr CreateExternalViewEmbedder() override; @@ -171,7 +170,6 @@ class PlatformView final : public flutter::PlatformView, // alive there const fuchsia::ui::views::ViewRef view_ref_; fuchsia::ui::views::FocuserPtr focuser_; - std::unique_ptr accessibility_bridge_; // Logical size and logical->physical ratio. These are optional to provide // an "unset" state during program startup, before Scenic has sent any @@ -190,6 +188,11 @@ class PlatformView final : public flutter::PlatformView, OnUpdateView on_update_view_callback_; OnDestroyView on_destroy_view_callback_; OnCreateSurface on_create_surface_callback_; + + // Accessibility handlers: + OnSemanticsNodeUpdate on_semantics_node_update_callback_; + OnRequestAnnounce on_request_announce_callback_; + std::shared_ptr external_view_embedder_; int current_text_input_client_ = 0; diff --git a/engine/src/flutter/shell/platform/fuchsia/flutter/platform_view_unittest.cc b/engine/src/flutter/shell/platform/fuchsia/flutter/platform_view_unittest.cc index 23850c860aa..5b241a3d691 100644 --- a/engine/src/flutter/shell/platform/fuchsia/flutter/platform_view_unittest.cc +++ b/engine/src/flutter/shell/platform/fuchsia/flutter/platform_view_unittest.cc @@ -268,8 +268,10 @@ class PlatformViewBuilder { std::move(on_create_view_callback_), std::move(on_update_view_callback_), std::move(on_destroy_view_callback_), - std::move(on_create_surface_callback_), view_embedder_, [](auto...) {}, - [](auto...) {}); + std::move(on_create_surface_callback_), + std::move(on_semantics_node_update_callback_), + std::move(on_request_announce_callback_), view_embedder_, + [](auto...) {}, [](auto...) {}); } private: @@ -298,6 +300,8 @@ class PlatformViewBuilder { OnUpdateView on_update_view_callback_{nullptr}; OnDestroyView on_destroy_view_callback_{nullptr}; OnCreateSurface on_create_surface_callback_{nullptr}; + OnSemanticsNodeUpdate on_semantics_node_update_callback_{nullptr}; + OnRequestAnnounce on_request_announce_callback_{nullptr}; std::shared_ptr view_embedder_{nullptr}; fml::TimeDelta vsync_offset_{fml::TimeDelta::Zero()}; };