From 9d98d886ad7ededc0162948c10eb95babffe2bb3 Mon Sep 17 00:00:00 2001 From: Travis Martin Date: Thu, 21 Sep 2017 12:41:13 -0700 Subject: [PATCH] Report onscreen text to Fuchsia context engine. (#4097) Add a class to track onscreen text via SemanticsNodes from the accessibility layer, and report this text to the ContextEngine. --- content_handler/BUILD.gn | 3 + content_handler/accessibility_bridge.cc | 83 +++++++++++++++++++++++++ content_handler/accessibility_bridge.h | 42 +++++++++++++ content_handler/runtime_holder.cc | 7 ++- content_handler/runtime_holder.h | 4 ++ travis/licenses_golden/licenses_flutter | 2 + 6 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 content_handler/accessibility_bridge.cc create mode 100644 content_handler/accessibility_bridge.h diff --git a/content_handler/BUILD.gn b/content_handler/BUILD.gn index 7b5ccba503d..902a38ba383 100644 --- a/content_handler/BUILD.gn +++ b/content_handler/BUILD.gn @@ -18,6 +18,8 @@ template("flutter_content_handler") { libs = [] sources = [ + "accessibility_bridge.cc", + "accessibility_bridge.h", "app.cc", "app.h", "application_controller_impl.cc", @@ -48,6 +50,7 @@ template("flutter_content_handler") { "//garnet/public/lib/ui/scenic:client", "//garnet/public/lib/ui/input/fidl", "//garnet/public/lib/ui/views/fidl", + "//apps/maxwell/services/context", "//dart/runtime/bin:libdart_builtin", "//dart/runtime/platform:libdart_platform", "//flutter/assets", diff --git a/content_handler/accessibility_bridge.cc b/content_handler/accessibility_bridge.cc new file mode 100644 index 00000000000..b14b34ccf32 --- /dev/null +++ b/content_handler/accessibility_bridge.cc @@ -0,0 +1,83 @@ +// Copyright 2017 The Fuchsia Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/content_handler/accessibility_bridge.h" + +#include + +#include "apps/maxwell/services/context/context_writer.fidl.h" +#include "flutter/lib/ui/semantics/semantics_node.h" +#include "lib/app/cpp/application_context.h" +#include "lib/fxl/macros.h" +#include "third_party/rapidjson/rapidjson/document.h" +#include "third_party/rapidjson/rapidjson/stringbuffer.h" +#include "third_party/rapidjson/rapidjson/writer.h" + +namespace flutter_runner { + +AccessibilityBridge::AccessibilityBridge(app::ApplicationContext* context) + : writer_(context->ConnectToEnvironmentService()) { + writer_.set_connection_error_handler( + [] { FXL_LOG(ERROR) << "Error connecting to ContextWriter."; }); +} + +void AccessibilityBridge::UpdateSemantics( + const std::vector& update) { + for (const auto& node : update) { + semantics_nodes_[node.id] = node; + } + std::vector visited_nodes; + UpdateVisitedForNodeAndChildren(0, &visited_nodes); + EraseUnvisitedNodes(visited_nodes); + + // The data sent to the Context Service is a JSON formatted list of labels + // for all on screen widgets. + rapidjson::Document nodes_json(rapidjson::kArrayType); + for (const int node_index : visited_nodes) { + const auto& node = semantics_nodes_[node_index]; + if (!node.label.empty()) { + rapidjson::Value value; + value.SetString(node.label.data(), node.label.size()); + nodes_json.PushBack(value, nodes_json.GetAllocator()); + } + } + + if (nodes_json.Size() > 0) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + nodes_json.Accept(writer); + writer_->WriteEntityTopic("/inferred/accessibility_text", + buffer.GetString()); + } +} + +void AccessibilityBridge::UpdateVisitedForNodeAndChildren( + const int id, + std::vector* visited_nodes) { + std::map::const_iterator it = + semantics_nodes_.find(id); + if (it == semantics_nodes_.end()) { + return; + } + + visited_nodes->push_back(id); + for (const int child : it->second.children) { + UpdateVisitedForNodeAndChildren(child, visited_nodes); + } +} + +void AccessibilityBridge::EraseUnvisitedNodes( + const std::vector& visited_nodes) { + const std::unordered_set visited_nodes_lookup(visited_nodes.begin(), + visited_nodes.end()); + for (auto it = semantics_nodes_.begin(); it != semantics_nodes_.end();) { + if (visited_nodes_lookup.find((*it).first) == visited_nodes_lookup.end()) { + it = semantics_nodes_.erase(it); + } else { + ++it; + } + } +} + +} // namespace flutter_runner diff --git a/content_handler/accessibility_bridge.h b/content_handler/accessibility_bridge.h new file mode 100644 index 00000000000..8de5bb0a4a4 --- /dev/null +++ b/content_handler/accessibility_bridge.h @@ -0,0 +1,42 @@ +// Copyright 2017 The Fuchsia Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_CONTENT_HANDLER_ACCESSIBILITY_BRIDGE_H_ +#define FLUTTER_CONTENT_HANDLER_ACCESSIBILITY_BRIDGE_H_ + +#include + +#include "apps/maxwell/services/context/context_writer.fidl.h" +#include "flutter/lib/ui/semantics/semantics_node.h" +#include "lib/app/cpp/application_context.h" + +namespace flutter_runner { + +// Maintain an up-to-date list of SemanticsNodes on screen, and communicate +// with the Context Service. +class AccessibilityBridge { + public: + explicit AccessibilityBridge(app::ApplicationContext* context); + + // Update the internal representation of the semantics nodes, and write the + // semantics to Context Service. + void UpdateSemantics(const std::vector& update); + + private: + // Walk the semantics node tree starting at |id|, and store the id of each + // visited child in |visited_nodes|. + void UpdateVisitedForNodeAndChildren(const int id, + std::vector* visited_nodes); + + // Remove any node from |semantics_nodes_| that doesn't have an id in + // |visited_nodes|. + void EraseUnvisitedNodes(const std::vector& visited_nodes); + + std::map semantics_nodes_; + maxwell::ContextWriterPtr writer_; +}; + +} // namespace flutter_runner + +#endif // FLUTTER_CONTENT_HANDLER_ACCESSIBILITY_BRIDGE_H_ diff --git a/content_handler/runtime_holder.cc b/content_handler/runtime_holder.cc index 18e0060fbd0..46eadb26fdb 100644 --- a/content_handler/runtime_holder.cc +++ b/content_handler/runtime_holder.cc @@ -13,6 +13,7 @@ #include "dart/runtime/include/dart_api.h" #include "flutter/assets/zip_asset_store.h" #include "flutter/common/threads.h" +#include "flutter/content_handler/accessibility_bridge.h" #include "flutter/content_handler/rasterizer.h" #include "flutter/content_handler/service_protocol_hooks.h" #include "flutter/lib/snapshot/snapshot.h" @@ -160,6 +161,8 @@ void RuntimeHolder::Init( blink::SetRegisterNativeServiceProtocolExtensionHook( ServiceProtocolHooks::RegisterHooks); } + + accessibility_bridge_ = std::make_unique(context_.get()); } void RuntimeHolder::CreateView( @@ -338,7 +341,9 @@ void RuntimeHolder::Render(std::unique_ptr layer_tree) { })); } -void RuntimeHolder::UpdateSemantics(std::vector update) {} +void RuntimeHolder::UpdateSemantics(std::vector update) { + accessibility_bridge_->UpdateSemantics(update); +} void RuntimeHolder::HandlePlatformMessage( fxl::RefPtr message) { diff --git a/content_handler/runtime_holder.h b/content_handler/runtime_holder.h index 93e9b22a037..86fd1b6e33d 100644 --- a/content_handler/runtime_holder.h +++ b/content_handler/runtime_holder.h @@ -13,6 +13,7 @@ #include "dart-pkg/fuchsia/sdk_ext/fuchsia.h" #include "flutter/assets/unzipper_provider.h" #include "flutter/assets/zip_asset_store.h" +#include "flutter/content_handler/accessibility_bridge.h" #include "flutter/flow/layers/layer_tree.h" #include "flutter/lib/ui/window/viewport_metrics.h" #include "flutter/runtime/runtime_controller.h" @@ -30,6 +31,7 @@ #include "lib/ui/views/fidl/view_manager.fidl.h" namespace flutter_runner { + class Rasterizer; class RuntimeHolder : public blink::RuntimeDelegate, @@ -124,6 +126,8 @@ class RuntimeHolder : public blink::RuntimeDelegate, fxl::WeakPtrFactory weak_factory_; + std::unique_ptr accessibility_bridge_; + FXL_DISALLOW_COPY_AND_ASSIGN(RuntimeHolder); }; diff --git a/travis/licenses_golden/licenses_flutter b/travis/licenses_golden/licenses_flutter index cca79fa332e..05614632b22 100644 --- a/travis/licenses_golden/licenses_flutter +++ b/travis/licenses_golden/licenses_flutter @@ -9280,6 +9280,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. LIBRARY: engine ORIGIN: ../../../garnet/LICENSE TYPE: LicenseType.bsd +FILE: ../../../flutter/content_handler/accessibility_bridge.cc +FILE: ../../../flutter/content_handler/accessibility_bridge.h FILE: ../../../flutter/content_handler/vulkan_surface.cc FILE: ../../../flutter/content_handler/vulkan_surface.h FILE: ../../../flutter/content_handler/vulkan_surface_pool.cc