Report onscreen text to Fuchsia context engine. (#4097)

Add a class to track onscreen text via SemanticsNodes from the
accessibility layer, and report this text to the ContextEngine.

content_handler/BUILD.gn

@@ -18,6 +18,8 @@ template("flutter_content_handler") {
     libs = []
 
     sources = [
+      "accessibility_bridge.cc",
+      "accessibility_bridge.h",
       "app.cc",
       "app.h",
       "application_controller_impl.cc",
@@ -48,6 +50,7 @@ template("flutter_content_handler") {
              "//garnet/public/lib/ui/scenic:client",
              "//garnet/public/lib/ui/input/fidl",
              "//garnet/public/lib/ui/views/fidl",
+             "//apps/maxwell/services/context",
              "//dart/runtime/bin:libdart_builtin",
              "//dart/runtime/platform:libdart_platform",
              "//flutter/assets",

content_handler/accessibility_bridge.cc

@@ -0,0 +1,83 @@
+// Copyright 2017 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "flutter/content_handler/accessibility_bridge.h"
+
+#include <unordered_set>
+
+#include "apps/maxwell/services/context/context_writer.fidl.h"
+#include "flutter/lib/ui/semantics/semantics_node.h"
+#include "lib/app/cpp/application_context.h"
+#include "lib/fxl/macros.h"
+#include "third_party/rapidjson/rapidjson/document.h"
+#include "third_party/rapidjson/rapidjson/stringbuffer.h"
+#include "third_party/rapidjson/rapidjson/writer.h"
+
+namespace flutter_runner {
+
+AccessibilityBridge::AccessibilityBridge(app::ApplicationContext* context)
+    : writer_(context->ConnectToEnvironmentService<maxwell::ContextWriter>()) {
+  writer_.set_connection_error_handler(
+      [] { FXL_LOG(ERROR) << "Error connecting to ContextWriter."; });
+}
+
+void AccessibilityBridge::UpdateSemantics(
+    const std::vector<blink::SemanticsNode>& update) {
+  for (const auto& node : update) {
+    semantics_nodes_[node.id] = node;
+  }
+  std::vector<int> visited_nodes;
+  UpdateVisitedForNodeAndChildren(0, &visited_nodes);
+  EraseUnvisitedNodes(visited_nodes);
+
+  // The data sent to the Context Service is a JSON formatted list of labels
+  // for all on screen widgets.
+  rapidjson::Document nodes_json(rapidjson::kArrayType);
+  for (const int node_index : visited_nodes) {
+    const auto& node = semantics_nodes_[node_index];
+    if (!node.label.empty()) {
+      rapidjson::Value value;
+      value.SetString(node.label.data(), node.label.size());
+      nodes_json.PushBack(value, nodes_json.GetAllocator());
+    }
+  }
+
+  if (nodes_json.Size() > 0) {
+    rapidjson::StringBuffer buffer;
+    rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
+    nodes_json.Accept(writer);
+    writer_->WriteEntityTopic("/inferred/accessibility_text",
+                              buffer.GetString());
+  }
+}
+
+void AccessibilityBridge::UpdateVisitedForNodeAndChildren(
+    const int id,
+    std::vector<int>* visited_nodes) {
+  std::map<int, blink::SemanticsNode>::const_iterator it =
+      semantics_nodes_.find(id);
+  if (it == semantics_nodes_.end()) {
+    return;
+  }
+
+  visited_nodes->push_back(id);
+  for (const int child : it->second.children) {
+    UpdateVisitedForNodeAndChildren(child, visited_nodes);
+  }
+}
+
+void AccessibilityBridge::EraseUnvisitedNodes(
+    const std::vector<int>& visited_nodes) {
+  const std::unordered_set<int> visited_nodes_lookup(visited_nodes.begin(),
+                                                     visited_nodes.end());
+  for (auto it = semantics_nodes_.begin(); it != semantics_nodes_.end();) {
+    if (visited_nodes_lookup.find((*it).first) == visited_nodes_lookup.end()) {
+      it = semantics_nodes_.erase(it);
+    } else {
+      ++it;
+    }
+  }
+}
+
+}  // namespace flutter_runner

content_handler/accessibility_bridge.h

@@ -0,0 +1,42 @@
+// Copyright 2017 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef FLUTTER_CONTENT_HANDLER_ACCESSIBILITY_BRIDGE_H_
+#define FLUTTER_CONTENT_HANDLER_ACCESSIBILITY_BRIDGE_H_
+
+#include <map>
+
+#include "apps/maxwell/services/context/context_writer.fidl.h"
+#include "flutter/lib/ui/semantics/semantics_node.h"
+#include "lib/app/cpp/application_context.h"
+
+namespace flutter_runner {
+
+// Maintain an up-to-date list of SemanticsNodes on screen, and communicate
+// with the Context Service.
+class AccessibilityBridge {
+ public:
+  explicit AccessibilityBridge(app::ApplicationContext* context);
+
+  // Update the internal representation of the semantics nodes, and write the
+  // semantics to Context Service.
+  void UpdateSemantics(const std::vector<blink::SemanticsNode>& update);
+
+ private:
+  // Walk the semantics node tree starting at |id|, and store the id of each
+  // visited child in |visited_nodes|.
+  void UpdateVisitedForNodeAndChildren(const int id,
+                                       std::vector<int>* visited_nodes);
+
+  // Remove any node from |semantics_nodes_| that doesn't have an id in
+  // |visited_nodes|.
+  void EraseUnvisitedNodes(const std::vector<int>& visited_nodes);
+
+  std::map<int, blink::SemanticsNode> semantics_nodes_;
+  maxwell::ContextWriterPtr writer_;
+};
+
+}  // namespace flutter_runner
+
+#endif  // FLUTTER_CONTENT_HANDLER_ACCESSIBILITY_BRIDGE_H_

content_handler/runtime_holder.cc

@@ -13,6 +13,7 @@
 #include "dart/runtime/include/dart_api.h"
 #include "flutter/assets/zip_asset_store.h"
 #include "flutter/common/threads.h"
+#include "flutter/content_handler/accessibility_bridge.h"
 #include "flutter/content_handler/rasterizer.h"
 #include "flutter/content_handler/service_protocol_hooks.h"
 #include "flutter/lib/snapshot/snapshot.h"
@@ -160,6 +161,8 @@ void RuntimeHolder::Init(
     blink::SetRegisterNativeServiceProtocolExtensionHook(
         ServiceProtocolHooks::RegisterHooks);
   }
+
+  accessibility_bridge_ = std::make_unique<AccessibilityBridge>(context_.get());
 }
 
 void RuntimeHolder::CreateView(
@@ -338,7 +341,9 @@ void RuntimeHolder::Render(std::unique_ptr<flow::LayerTree> layer_tree) {
   }));
 }
 
-void RuntimeHolder::UpdateSemantics(std::vector<blink::SemanticsNode> update) {}
+void RuntimeHolder::UpdateSemantics(std::vector<blink::SemanticsNode> update) {
+  accessibility_bridge_->UpdateSemantics(update);
+}
 
 void RuntimeHolder::HandlePlatformMessage(
     fxl::RefPtr<blink::PlatformMessage> message) {

content_handler/runtime_holder.h

@@ -13,6 +13,7 @@
 #include "dart-pkg/fuchsia/sdk_ext/fuchsia.h"
 #include "flutter/assets/unzipper_provider.h"
 #include "flutter/assets/zip_asset_store.h"
+#include "flutter/content_handler/accessibility_bridge.h"
 #include "flutter/flow/layers/layer_tree.h"
 #include "flutter/lib/ui/window/viewport_metrics.h"
 #include "flutter/runtime/runtime_controller.h"
@@ -30,6 +31,7 @@
 #include "lib/ui/views/fidl/view_manager.fidl.h"
 
 namespace flutter_runner {
+
 class Rasterizer;
 
 class RuntimeHolder : public blink::RuntimeDelegate,
@@ -124,6 +126,8 @@ class RuntimeHolder : public blink::RuntimeDelegate,
 
   fxl::WeakPtrFactory<RuntimeHolder> weak_factory_;
 
+  std::unique_ptr<AccessibilityBridge> accessibility_bridge_;
+
   FXL_DISALLOW_COPY_AND_ASSIGN(RuntimeHolder);
 };
 

travis/licenses_golden/licenses_flutter

@@ -9280,6 +9280,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 LIBRARY: engine
 ORIGIN: ../../../garnet/LICENSE
 TYPE: LicenseType.bsd
+FILE: ../../../flutter/content_handler/accessibility_bridge.cc
+FILE: ../../../flutter/content_handler/accessibility_bridge.h
 FILE: ../../../flutter/content_handler/vulkan_surface.cc
 FILE: ../../../flutter/content_handler/vulkan_surface.h
 FILE: ../../../flutter/content_handler/vulkan_surface_pool.cc