From 5bf14b1e4d8dcf7fe3e12aabe7dedb4cd326b4e6 Mon Sep 17 00:00:00 2001 From: Jonah Williams Date: Wed, 11 Jul 2018 10:27:50 -0700 Subject: [PATCH] Custom accessibility (local context) action support for iOS and Android. (flutter/engine#5597) --- engine/src/flutter/lib/ui/BUILD.gn | 2 + engine/src/flutter/lib/ui/semantics.dart | 24 +++++++ .../semantics/custom_accessibility_action.cc | 13 ++++ .../semantics/custom_accessibility_action.h | 35 +++++++++ .../flutter/lib/ui/semantics/semantics_node.h | 2 + .../lib/ui/semantics/semantics_update.cc | 14 ++-- .../lib/ui/semantics/semantics_update.h | 10 ++- .../ui/semantics/semantics_update_builder.cc | 20 ++++-- .../ui/semantics/semantics_update_builder.h | 7 +- .../src/flutter/runtime/runtime_controller.cc | 2 +- engine/src/flutter/runtime/runtime_delegate.h | 4 +- engine/src/flutter/shell/common/engine.cc | 5 +- engine/src/flutter/shell/common/engine.h | 7 +- .../src/flutter/shell/common/platform_view.cc | 3 +- .../src/flutter/shell/common/platform_view.h | 4 +- engine/src/flutter/shell/common/shell.cc | 7 +- engine/src/flutter/shell/common/shell.h | 4 +- .../io/flutter/view/AccessibilityBridge.java | 72 ++++++++++++++++++- .../io/flutter/view/FlutterNativeView.java | 7 ++ .../android/io/flutter/view/FlutterView.java | 11 +++ .../platform/android/platform_view_android.cc | 42 +++++++++-- .../platform/android/platform_view_android.h | 3 +- .../android/platform_view_android_jni.cc | 17 +++++ .../android/platform_view_android_jni.h | 5 ++ .../framework/Source/accessibility_bridge.h | 19 ++++- .../framework/Source/accessibility_bridge.mm | 47 +++++++++++- .../platform/darwin/ios/platform_view_ios.h | 3 +- .../platform/darwin/ios/platform_view_ios.mm | 5 +- 28 files changed, 359 insertions(+), 35 deletions(-) create mode 100644 engine/src/flutter/lib/ui/semantics/custom_accessibility_action.cc create mode 100644 engine/src/flutter/lib/ui/semantics/custom_accessibility_action.h diff --git a/engine/src/flutter/lib/ui/BUILD.gn b/engine/src/flutter/lib/ui/BUILD.gn index c8f7e1fb21d..8526070f9b1 100644 --- a/engine/src/flutter/lib/ui/BUILD.gn +++ b/engine/src/flutter/lib/ui/BUILD.gn @@ -58,6 +58,8 @@ source_set("ui") { "semantics/semantics_update.h", "semantics/semantics_update_builder.cc", "semantics/semantics_update_builder.h", + "semantics/custom_accessibility_action.cc", + "semantics/custom_accessibility_action.h", "text/asset_manager_font_provider.cc", "text/asset_manager_font_provider.h", "text/font_collection.cc", diff --git a/engine/src/flutter/lib/ui/semantics.dart b/engine/src/flutter/lib/ui/semantics.dart index 86f6d04abb0..174b21ae8d2 100644 --- a/engine/src/flutter/lib/ui/semantics.dart +++ b/engine/src/flutter/lib/ui/semantics.dart @@ -26,6 +26,7 @@ class SemanticsAction { static const int _kPasteIndex = 1 << 14; static const int _kDidGainAccessibilityFocusIndex = 1 << 15; static const int _kDidLoseAccessibilityFocusIndex = 1 << 16; + static const int _kCustomAction = 1 << 17; /// The numerical value for this action. /// @@ -146,6 +147,12 @@ class SemanticsAction { /// Accessibility focus and input focus can be held by two different nodes! static const SemanticsAction didLoseAccessibilityFocus = const SemanticsAction._(_kDidLoseAccessibilityFocusIndex); + /// Indicates that the user has invoked a custom accessibility action. + /// + /// This handler is added automatically whenever a custom accessibility + /// action is added to a semantics node. + static const SemanticsAction customAction = const SemanticsAction._(_kCustomAction); + /// The possible semantics actions. /// /// The map's key is the [index] of the action and the value is the action @@ -168,6 +175,7 @@ class SemanticsAction { _kPasteIndex: paste, _kDidGainAccessibilityFocusIndex: didGainAccessibilityFocus, _kDidLoseAccessibilityFocusIndex: didLoseAccessibilityFocus, + _kCustomAction: customAction, }; @override @@ -207,6 +215,8 @@ class SemanticsAction { return 'SemanticsAction.didGainAccessibilityFocus'; case _kDidLoseAccessibilityFocusIndex: return 'SemanticsAction.didLoseAccessibilityFocus'; + case _kCustomAction: + return 'SemanticsAction.customAction'; } return null; } @@ -498,6 +508,7 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 { Float64List transform, Int32List childrenInTraversalOrder, Int32List childrenInHitTestOrder, + Int32List customAcccessibilityActions, }) { if (transform.length != 16) throw new ArgumentError('transform argument must have 16 entries.'); @@ -523,6 +534,7 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 { transform, childrenInTraversalOrder, childrenInHitTestOrder, + customAcccessibilityActions, ); } void _updateNode( @@ -547,8 +559,20 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 { Float64List transform, Int32List childrenInTraversalOrder, Int32List childrenInHitTestOrder, + Int32List customAcccessibilityActions, ) native 'SemanticsUpdateBuilder_updateNode'; + /// Update the custom accessibility action associated with the given `id`. + /// + /// The name of the action exposed to the user is the `label`. The text + /// direction of this label is the same as the global window. + void updateCustomAction({int id, String label}) { + assert(id != null); + assert(label != null && label != ''); + _updateCustomAction(id, label); + } + void _updateCustomAction(int id, String label) native 'SemanticsUpdateBuilder_updateAction'; + /// Creates a [SemanticsUpdate] object that encapsulates the updates recorded /// by this object. /// diff --git a/engine/src/flutter/lib/ui/semantics/custom_accessibility_action.cc b/engine/src/flutter/lib/ui/semantics/custom_accessibility_action.cc new file mode 100644 index 00000000000..7f404e81702 --- /dev/null +++ b/engine/src/flutter/lib/ui/semantics/custom_accessibility_action.cc @@ -0,0 +1,13 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/lib/ui/semantics/custom_accessibility_action.h" + +namespace blink { + +CustomAccessibilityAction::CustomAccessibilityAction() = default; + +CustomAccessibilityAction::~CustomAccessibilityAction() = default; + +} diff --git a/engine/src/flutter/lib/ui/semantics/custom_accessibility_action.h b/engine/src/flutter/lib/ui/semantics/custom_accessibility_action.h new file mode 100644 index 00000000000..9ccfd4839fd --- /dev/null +++ b/engine/src/flutter/lib/ui/semantics/custom_accessibility_action.h @@ -0,0 +1,35 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_LIB_UI_SEMANTICS_CUSTOM_ACCESSIBILITY_ACTION_H_ +#define FLUTTER_LIB_UI_SEMANTICS_CUSTOM_ACCESSIBILITY_ACTION_H_ + +#include "lib/tonic/dart_wrappable.h" +#include "lib/tonic/typed_data/float64_list.h" +#include "lib/tonic/typed_data/int32_list.h" +#include "lib/tonic/dart_library_natives.h" + +namespace blink { + +/// A custom accessibility action is used to indicate additional semantics +/// actions that a user can perform on a semantics node beyond the +/// preconfigured options. +struct CustomAccessibilityAction { + CustomAccessibilityAction(); + ~CustomAccessibilityAction(); + + int32_t id = 0; + std::string label; +}; + + +// Contains custom accessibility actions that need to be updated. +// +// The keys in the map are stable action IDs, and the values contain +// semantic information for the action corresponding to that id. +using CustomAccessibilityActionUpdates = std::unordered_map; + +} // namespace blink + +#endif //FLUTTER_LIB_UI_SEMANTICS_LOCAL_CONTEXT_ACTION_H_ diff --git a/engine/src/flutter/lib/ui/semantics/semantics_node.h b/engine/src/flutter/lib/ui/semantics/semantics_node.h index 3ddb431e90d..e0ae2ba296c 100644 --- a/engine/src/flutter/lib/ui/semantics/semantics_node.h +++ b/engine/src/flutter/lib/ui/semantics/semantics_node.h @@ -35,6 +35,7 @@ enum class SemanticsAction : int32_t { kPaste = 1 << 14, kDidGainAccessibilityFocus = 1 << 15, kDidLoseAccessibilityFocus = 1 << 16, + kCustomAction = 1 << 17, }; const int kScrollableSemanticsActions = @@ -87,6 +88,7 @@ struct SemanticsNode { SkMatrix44 transform = SkMatrix44(SkMatrix44::kIdentity_Constructor); std::vector childrenInTraversalOrder; std::vector childrenInHitTestOrder; + std::vector customAccessibilityActions; }; // Contains semantic nodes that need to be updated. diff --git a/engine/src/flutter/lib/ui/semantics/semantics_update.cc b/engine/src/flutter/lib/ui/semantics/semantics_update.cc index fdc796e937b..4905d95f774 100644 --- a/engine/src/flutter/lib/ui/semantics/semantics_update.cc +++ b/engine/src/flutter/lib/ui/semantics/semantics_update.cc @@ -21,12 +21,14 @@ IMPLEMENT_WRAPPERTYPEINFO(ui, SemanticsUpdate); DART_BIND_ALL(SemanticsUpdate, FOR_EACH_BINDING) fxl::RefPtr SemanticsUpdate::create( - SemanticsNodeUpdates nodes) { - return fxl::MakeRefCounted(std::move(nodes)); + SemanticsNodeUpdates nodes, + CustomAccessibilityActionUpdates actions) { + return fxl::MakeRefCounted(std::move(nodes), std::move(actions)); } -SemanticsUpdate::SemanticsUpdate(SemanticsNodeUpdates nodes) - : nodes_(std::move(nodes)) {} +SemanticsUpdate::SemanticsUpdate(SemanticsNodeUpdates nodes, + CustomAccessibilityActionUpdates actions) + : nodes_(std::move(nodes)), actions_(std::move(actions)) {} SemanticsUpdate::~SemanticsUpdate() = default; @@ -34,6 +36,10 @@ SemanticsNodeUpdates SemanticsUpdate::takeNodes() { return std::move(nodes_); } +CustomAccessibilityActionUpdates SemanticsUpdate::takeActions() { + return std::move(actions_); +} + void SemanticsUpdate::dispose() { ClearDartWrapper(); } diff --git a/engine/src/flutter/lib/ui/semantics/semantics_update.h b/engine/src/flutter/lib/ui/semantics/semantics_update.h index dbea854e4a0..f6773239f41 100644 --- a/engine/src/flutter/lib/ui/semantics/semantics_update.h +++ b/engine/src/flutter/lib/ui/semantics/semantics_update.h @@ -6,6 +6,7 @@ #define FLUTTER_LIB_UI_SEMANTICS_SEMANTICS_UPDATE_H_ #include "flutter/lib/ui/semantics/semantics_node.h" +#include "flutter/lib/ui/semantics/custom_accessibility_action.h" #include "lib/tonic/dart_wrappable.h" namespace tonic { @@ -21,18 +22,23 @@ class SemanticsUpdate : public fxl::RefCountedThreadSafe, public: ~SemanticsUpdate() override; - static fxl::RefPtr create(SemanticsNodeUpdates nodes); + static fxl::RefPtr create(SemanticsNodeUpdates nodes, + CustomAccessibilityActionUpdates actions); SemanticsNodeUpdates takeNodes(); + CustomAccessibilityActionUpdates takeActions(); + void dispose(); static void RegisterNatives(tonic::DartLibraryNatives* natives); private: - explicit SemanticsUpdate(SemanticsNodeUpdates nodes); + explicit SemanticsUpdate(SemanticsNodeUpdates nodes, + CustomAccessibilityActionUpdates updates); SemanticsNodeUpdates nodes_; + CustomAccessibilityActionUpdates actions_; }; } // namespace blink diff --git a/engine/src/flutter/lib/ui/semantics/semantics_update_builder.cc b/engine/src/flutter/lib/ui/semantics/semantics_update_builder.cc index 576967525b4..c39423a358a 100644 --- a/engine/src/flutter/lib/ui/semantics/semantics_update_builder.cc +++ b/engine/src/flutter/lib/ui/semantics/semantics_update_builder.cc @@ -17,8 +17,9 @@ static void SemanticsUpdateBuilder_constructor(Dart_NativeArguments args) { IMPLEMENT_WRAPPERTYPEINFO(ui, SemanticsUpdateBuilder); -#define FOR_EACH_BINDING(V) \ - V(SemanticsUpdateBuilder, updateNode) \ +#define FOR_EACH_BINDING(V) \ + V(SemanticsUpdateBuilder, updateNode) \ + V(SemanticsUpdateBuilder, updateCustomAction) \ V(SemanticsUpdateBuilder, build) FOR_EACH_BINDING(DART_NATIVE_CALLBACK) @@ -54,7 +55,8 @@ void SemanticsUpdateBuilder::updateNode(int id, int textDirection, const tonic::Float64List& transform, const tonic::Int32List& childrenInTraversalOrder, - const tonic::Int32List& childrenInHitTestOrder) { + const tonic::Int32List& childrenInHitTestOrder, + const tonic::Int32List& localContextActions) { SemanticsNode node; node.id = id; node.flags = flags; @@ -76,11 +78,21 @@ void SemanticsUpdateBuilder::updateNode(int id, childrenInTraversalOrder.data(), childrenInTraversalOrder.data() + childrenInTraversalOrder.num_elements()); node.childrenInHitTestOrder = std::vector( childrenInHitTestOrder.data(), childrenInHitTestOrder.data() + childrenInHitTestOrder.num_elements()); + node.customAccessibilityActions = std::vector( + localContextActions.data(), localContextActions.data() + localContextActions.num_elements()); nodes_[id] = node; } +void SemanticsUpdateBuilder::updateCustomAction(int id, + std::string label) { + CustomAccessibilityAction action; + action.id = id; + action.label = label; + actions_[id] = action; +} + fxl::RefPtr SemanticsUpdateBuilder::build() { - return SemanticsUpdate::create(std::move(nodes_)); + return SemanticsUpdate::create(std::move(nodes_), std::move(actions_)); } } // namespace blink diff --git a/engine/src/flutter/lib/ui/semantics/semantics_update_builder.h b/engine/src/flutter/lib/ui/semantics/semantics_update_builder.h index c4d4b277816..dca0f0beaca 100644 --- a/engine/src/flutter/lib/ui/semantics/semantics_update_builder.h +++ b/engine/src/flutter/lib/ui/semantics/semantics_update_builder.h @@ -45,7 +45,11 @@ class SemanticsUpdateBuilder int textDirection, const tonic::Float64List& transform, const tonic::Int32List& childrenInTraversalOrder, - const tonic::Int32List& childrenInHitTestOrder); + const tonic::Int32List& childrenInHitTestOrder, + const tonic::Int32List& customAccessibilityActions); + + void updateCustomAction(int id, + std::string label); fxl::RefPtr build(); @@ -55,6 +59,7 @@ class SemanticsUpdateBuilder explicit SemanticsUpdateBuilder(); SemanticsNodeUpdates nodes_; + CustomAccessibilityActionUpdates actions_; }; } // namespace blink diff --git a/engine/src/flutter/runtime/runtime_controller.cc b/engine/src/flutter/runtime/runtime_controller.cc index c1acc813f32..5ef6e5a0a03 100644 --- a/engine/src/flutter/runtime/runtime_controller.cc +++ b/engine/src/flutter/runtime/runtime_controller.cc @@ -240,7 +240,7 @@ void RuntimeController::Render(Scene* scene) { void RuntimeController::UpdateSemantics(SemanticsUpdate* update) { if (window_data_.semantics_enabled) { - client_.UpdateSemantics(update->takeNodes()); + client_.UpdateSemantics(update->takeNodes(), update->takeActions()); } } diff --git a/engine/src/flutter/runtime/runtime_delegate.h b/engine/src/flutter/runtime/runtime_delegate.h index 75e89e62e44..463ebbbd280 100644 --- a/engine/src/flutter/runtime/runtime_delegate.h +++ b/engine/src/flutter/runtime/runtime_delegate.h @@ -10,6 +10,7 @@ #include "flutter/flow/layers/layer_tree.h" #include "flutter/lib/ui/semantics/semantics_node.h" +#include "flutter/lib/ui/semantics/custom_accessibility_action.h" #include "flutter/lib/ui/text/font_collection.h" #include "flutter/lib/ui/window/platform_message.h" #include "third_party/dart/runtime/include/dart_api.h" @@ -24,7 +25,8 @@ class RuntimeDelegate { virtual void Render(std::unique_ptr layer_tree) = 0; - virtual void UpdateSemantics(blink::SemanticsNodeUpdates update) = 0; + virtual void UpdateSemantics(blink::SemanticsNodeUpdates update, + blink::CustomAccessibilityActionUpdates actions) = 0; virtual void HandlePlatformMessage(fxl::RefPtr message) = 0; diff --git a/engine/src/flutter/shell/common/engine.cc b/engine/src/flutter/shell/common/engine.cc index f06c155dbcb..60453633b12 100644 --- a/engine/src/flutter/shell/common/engine.cc +++ b/engine/src/flutter/shell/common/engine.cc @@ -364,8 +364,9 @@ void Engine::Render(std::unique_ptr layer_tree) { animator_->Render(std::move(layer_tree)); } -void Engine::UpdateSemantics(blink::SemanticsNodeUpdates update) { - delegate_.OnEngineUpdateSemantics(*this, std::move(update)); +void Engine::UpdateSemantics(blink::SemanticsNodeUpdates update, + blink::CustomAccessibilityActionUpdates actions) { + delegate_.OnEngineUpdateSemantics(*this, std::move(update), std::move(actions)); } void Engine::HandlePlatformMessage( diff --git a/engine/src/flutter/shell/common/engine.h b/engine/src/flutter/shell/common/engine.h index 0bbb524aae1..eeacfc4b848 100644 --- a/engine/src/flutter/shell/common/engine.h +++ b/engine/src/flutter/shell/common/engine.h @@ -11,6 +11,7 @@ #include "flutter/assets/asset_manager.h" #include "flutter/common/task_runners.h" #include "flutter/lib/ui/semantics/semantics_node.h" +#include "flutter/lib/ui/semantics/custom_accessibility_action.h" #include "flutter/lib/ui/text/font_collection.h" #include "flutter/lib/ui/window/platform_message.h" #include "flutter/lib/ui/window/viewport_metrics.h" @@ -32,7 +33,8 @@ class Engine final : public blink::RuntimeDelegate { public: virtual void OnEngineUpdateSemantics( const Engine& engine, - blink::SemanticsNodeUpdates update) = 0; + blink::SemanticsNodeUpdates update, + blink::CustomAccessibilityActionUpdates actions) = 0; virtual void OnEngineHandlePlatformMessage( const Engine& engine, @@ -123,7 +125,8 @@ class Engine final : public blink::RuntimeDelegate { void Render(std::unique_ptr layer_tree) override; // |blink::RuntimeDelegate| - void UpdateSemantics(blink::SemanticsNodeUpdates update) override; + void UpdateSemantics(blink::SemanticsNodeUpdates update, + blink::CustomAccessibilityActionUpdates actions) override; // |blink::RuntimeDelegate| void HandlePlatformMessage( diff --git a/engine/src/flutter/shell/common/platform_view.cc b/engine/src/flutter/shell/common/platform_view.cc index 53cf7b73668..cb84fbcedd7 100644 --- a/engine/src/flutter/shell/common/platform_view.cc +++ b/engine/src/flutter/shell/common/platform_view.cc @@ -74,7 +74,8 @@ fml::WeakPtr PlatformView::GetWeakPtr() const { return weak_factory_.GetWeakPtr(); } -void PlatformView::UpdateSemantics(blink::SemanticsNodeUpdates update) {} +void PlatformView::UpdateSemantics(blink::SemanticsNodeUpdates update, + blink::CustomAccessibilityActionUpdates actions) {} void PlatformView::HandlePlatformMessage( fxl::RefPtr message) { diff --git a/engine/src/flutter/shell/common/platform_view.h b/engine/src/flutter/shell/common/platform_view.h index 3ae91b178c7..8273563f132 100644 --- a/engine/src/flutter/shell/common/platform_view.h +++ b/engine/src/flutter/shell/common/platform_view.h @@ -11,6 +11,7 @@ #include "flutter/flow/texture.h" #include "flutter/fml/memory/weak_ptr.h" #include "flutter/lib/ui/semantics/semantics_node.h" +#include "flutter/lib/ui/semantics/custom_accessibility_action.h" #include "flutter/lib/ui/window/platform_message.h" #include "flutter/lib/ui/window/pointer_data_packet.h" #include "flutter/lib/ui/window/viewport_metrics.h" @@ -95,7 +96,8 @@ class PlatformView { fml::WeakPtr GetWeakPtr() const; - virtual void UpdateSemantics(blink::SemanticsNodeUpdates update); + virtual void UpdateSemantics(blink::SemanticsNodeUpdates updates, + blink::CustomAccessibilityActionUpdates actions); virtual void HandlePlatformMessage( fxl::RefPtr message); diff --git a/engine/src/flutter/shell/common/shell.cc b/engine/src/flutter/shell/common/shell.cc index 0da5126e53e..e6cb6fc10a8 100644 --- a/engine/src/flutter/shell/common/shell.cc +++ b/engine/src/flutter/shell/common/shell.cc @@ -707,14 +707,15 @@ void Shell::OnAnimatorDrawLastLayerTree(const Animator& animator) { // |shell::Engine::Delegate| void Shell::OnEngineUpdateSemantics(const Engine& engine, - blink::SemanticsNodeUpdates update) { + blink::SemanticsNodeUpdates update, + blink::CustomAccessibilityActionUpdates actions) { FXL_DCHECK(is_setup_); FXL_DCHECK(task_runners_.GetUITaskRunner()->RunsTasksOnCurrentThread()); task_runners_.GetPlatformTaskRunner()->PostTask( - [view = platform_view_->GetWeakPtr(), update = std::move(update)] { + [view = platform_view_->GetWeakPtr(), update = std::move(update), actions = std::move(actions)] { if (view) { - view->UpdateSemantics(std::move(update)); + view->UpdateSemantics(std::move(update), std::move(actions)); } }); } diff --git a/engine/src/flutter/shell/common/shell.h b/engine/src/flutter/shell/common/shell.h index 75ab39052ce..8f12bf0ae64 100644 --- a/engine/src/flutter/shell/common/shell.h +++ b/engine/src/flutter/shell/common/shell.h @@ -15,6 +15,7 @@ #include "flutter/fml/memory/weak_ptr.h" #include "flutter/fml/thread.h" #include "flutter/lib/ui/semantics/semantics_node.h" +#include "flutter/lib/ui/semantics/custom_accessibility_action.h" #include "flutter/lib/ui/window/platform_message.h" #include "flutter/runtime/service_protocol.h" #include "flutter/shell/common/animator.h" @@ -183,7 +184,8 @@ class Shell final : public PlatformView::Delegate, // |shell::Engine::Delegate| void OnEngineUpdateSemantics(const Engine& engine, - blink::SemanticsNodeUpdates update) override; + blink::SemanticsNodeUpdates update, + blink::CustomAccessibilityActionUpdates actions) override; // |shell::Engine::Delegate| void OnEngineHandlePlatformMessage( diff --git a/engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java b/engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java index 8925d6957cb..b0dddf8598c 100644 --- a/engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java +++ b/engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java @@ -32,6 +32,7 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess private static final int ROOT_NODE_ID = 0; private Map mObjects; + private Map mCustomAccessibilityActions; private final FlutterView mOwner; private boolean mAccessibilityEnabled = false; private SemanticsObject mA11yFocusedObject; @@ -59,7 +60,8 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess CUT(1 << 13), PASTE(1 << 14), DID_GAIN_ACCESSIBILITY_FOCUS(1 << 15), - DID_LOSE_ACCESSIBILITY_FOCUS(1 << 16); + DID_LOSE_ACCESSIBILITY_FOCUS(1 << 16), + CUSTOM_ACTION(1 << 17); Action(int value) { this.value = value; @@ -95,6 +97,7 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess assert owner != null; mOwner = owner; mObjects = new HashMap(); + mCustomAccessibilityActions = new HashMap(); previousRoutes = new ArrayList<>(); mFlutterAccessibilityChannel = new BasicMessageChannel<>(owner, "flutter/accessibility", StandardMessageCodec.INSTANCE); @@ -250,6 +253,15 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess result.addAction(AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS); } + // Actions on the local context menu + if (Build.VERSION.SDK_INT >= 21) { + if (object.customAccessibilityAction != null) { + for (CustomAccessibilityAction action : object.customAccessibilityAction) { + result.addAction(new AccessibilityNodeInfo.AccessibilityAction(action.resourceId, action.label)); + } + } + } + if (object.childrenInTraversalOrder != null) { for (SemanticsObject child : object.childrenInTraversalOrder) { if (!child.hasFlag(Flag.IS_HIDDEN)) { @@ -381,6 +393,14 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess mOwner.dispatchSemanticsAction(virtualViewId, Action.PASTE); return true; } + default: + // might be a custom accessibility action. + final int flutterId = action - firstResourceId; + CustomAccessibilityAction contextAction = mCustomAccessibilityActions.get(flutterId); + if (contextAction != null) { + mOwner.dispatchSemanticsAction(virtualViewId, Action.CUSTOM_ACTION, contextAction.id); + return true; + } } return false; } @@ -440,6 +460,17 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess return object; } + private CustomAccessibilityAction getOrCreateAction(int id) { + CustomAccessibilityAction action = mCustomAccessibilityActions.get(id); + if (action == null) { + action = new CustomAccessibilityAction(); + action.id = id; + action.resourceId = id + firstResourceId; + mCustomAccessibilityActions.put(id, action); + } + return action; + } + void handleTouchExplorationExit() { if (mHoveredObject != null) { sendAccessibilityEvent(mHoveredObject.id, AccessibilityEvent.TYPE_VIEW_HOVER_EXIT); @@ -464,6 +495,16 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess } } + void updateCustomAccessibilityActions(ByteBuffer buffer, String[] strings) { + ArrayList updatedActions = new ArrayList(); + while (buffer.hasRemaining()) { + int id = buffer.getInt(); + CustomAccessibilityAction action = getOrCreateAction(id); + int stringIndex = buffer.getInt(); + action.label = stringIndex == -1 ? null : strings[stringIndex]; + } + } + void updateSemantics(ByteBuffer buffer, String[] strings) { ArrayList updated = new ArrayList(); while (buffer.hasRemaining()) { @@ -732,6 +773,20 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess } } + private class CustomAccessibilityAction { + CustomAccessibilityAction() {} + + /// Resource id is the id of the custom action plus a minimum value so that the identifier + /// does not collide with existing Android accessibility actions. + int resourceId = -1; + int id = -1; + + /// The label is the user presented value which is displayed in the local context menu. + String label; + } + /// Value is derived from ACTION_TYPE_MASK in AccessibilityNodeInfo.java + static int firstResourceId = 267386881; + private class SemanticsObject { SemanticsObject() { } @@ -770,6 +825,7 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess SemanticsObject parent; List childrenInTraversalOrder; List childrenInHitTestOrder; + List customAccessibilityAction; private boolean inverseTransformDirty = true; private float[] inverseTransform; @@ -888,6 +944,20 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess childrenInHitTestOrder.add(child); } } + final int actionCount = buffer.getInt(); + if (actionCount == 0) { + customAccessibilityAction = null; + } else { + if (customAccessibilityAction == null) + customAccessibilityAction = new ArrayList(actionCount); + else + customAccessibilityAction.clear(); + + for (int i = 0; i < actionCount; i++) { + CustomAccessibilityAction action = getOrCreateAction(buffer.getInt()); + customAccessibilityAction.add(action); + } + } } private void ensureInverseTransform() { diff --git a/engine/src/flutter/shell/platform/android/io/flutter/view/FlutterNativeView.java b/engine/src/flutter/shell/platform/android/io/flutter/view/FlutterNativeView.java index 3d00312a081..c9899ee261e 100644 --- a/engine/src/flutter/shell/platform/android/io/flutter/view/FlutterNativeView.java +++ b/engine/src/flutter/shell/platform/android/io/flutter/view/FlutterNativeView.java @@ -183,6 +183,13 @@ public class FlutterNativeView implements BinaryMessenger { mFlutterView.updateSemantics(buffer, strings); } + // Called by native to update the custom accessibility actions. + private void updateCustomAccessibilityActions(ByteBuffer buffer, String[] strings) { + if (mFlutterView == null) + return; + mFlutterView.updateCustomAccessibilityActions(buffer, strings); + } + // Called by native to notify first Flutter frame rendered. private void onFirstFrame() { if (mFlutterView == null) diff --git a/engine/src/flutter/shell/platform/android/io/flutter/view/FlutterView.java b/engine/src/flutter/shell/platform/android/io/flutter/view/FlutterView.java index 703894a6147..8974a0bec5e 100644 --- a/engine/src/flutter/shell/platform/android/io/flutter/view/FlutterView.java +++ b/engine/src/flutter/shell/platform/android/io/flutter/view/FlutterView.java @@ -712,6 +712,17 @@ public class FlutterView extends SurfaceView } } + public void updateCustomAccessibilityActions(ByteBuffer buffer, String[] strings) { + try { + if (mAccessibilityNodeProvider != null) { + buffer.order(ByteOrder.LITTLE_ENDIAN); + mAccessibilityNodeProvider.updateCustomAccessibilityActions(buffer, strings); + } + } catch (Exception ex) { + Log.e(TAG, "Uncaught exception while updating local context actions", ex); + } + } + // Called by native to notify first Flutter frame rendered. public void onFirstFrame() { // Allow listeners to remove themselves when they are called. diff --git a/engine/src/flutter/shell/platform/android/platform_view_android.cc b/engine/src/flutter/shell/platform/android/platform_view_android.cc index a13d9bed149..422358da27f 100644 --- a/engine/src/flutter/shell/platform/android/platform_view_android.cc +++ b/engine/src/flutter/shell/platform/android/platform_view_android.cc @@ -178,9 +178,11 @@ void PlatformViewAndroid::DispatchSemanticsAction(JNIEnv* env, } // |shell::PlatformView| -void PlatformViewAndroid::UpdateSemantics(blink::SemanticsNodeUpdates update) { - constexpr size_t kBytesPerNode = 35 * sizeof(int32_t); +void PlatformViewAndroid::UpdateSemantics(blink::SemanticsNodeUpdates update, + blink::CustomAccessibilityActionUpdates actions) { + constexpr size_t kBytesPerNode = 36 * sizeof(int32_t); constexpr size_t kBytesPerChild = sizeof(int32_t); + constexpr size_t kBytesPerAction = 2 * sizeof(int32_t); JNIEnv* env = fml::jni::AttachCurrentThread(); { @@ -194,6 +196,7 @@ void PlatformViewAndroid::UpdateSemantics(blink::SemanticsNodeUpdates update) { num_bytes += value.second.childrenInTraversalOrder.size() * kBytesPerChild; num_bytes += value.second.childrenInHitTestOrder.size() * kBytesPerChild; + num_bytes += value.second.customAccessibilityActions.size() * kBytesPerChild; } std::vector buffer(num_bytes); @@ -259,14 +262,45 @@ void PlatformViewAndroid::UpdateSemantics(blink::SemanticsNodeUpdates update) { for (int32_t child : node.childrenInHitTestOrder) buffer_int32[position++] = child; + + buffer_int32[position++] = node.customAccessibilityActions.size(); + for (int32_t child : node.customAccessibilityActions) + buffer_int32[position++] = child; } + // custom accessibility actions. + size_t num_action_bytes = actions.size() * kBytesPerAction; + std::vector actions_buffer(num_action_bytes); + int32_t* actions_buffer_int32 = reinterpret_cast(&actions_buffer[0]); + + std::vector action_strings; + size_t actions_position = 0; + for (const auto& value : actions) { + // If you edit this code, make sure you update kBytesPerAction + // to match the number of values you are + // sending. + const blink::CustomAccessibilityAction& action = value.second; + actions_buffer_int32[actions_position++] = action.id; + if (action.label.empty()) { + actions_buffer_int32[actions_position++] = -1; + } else { + actions_buffer_int32[actions_position++] = action_strings.size(); + action_strings.push_back(action.label); + } + } + + fml::jni::ScopedJavaLocalRef direct_actions_buffer( + env, env->NewDirectByteBuffer(actions_buffer.data(), actions_buffer.size())); + fml::jni::ScopedJavaLocalRef direct_buffer( env, env->NewDirectByteBuffer(buffer.data(), buffer.size())); + FlutterViewUpdateCustomAccessibilityActions( + env, view.obj(), direct_actions_buffer.obj(), + fml::jni::VectorToStringArray(env, action_strings).obj()); FlutterViewUpdateSemantics( - env, view.obj(), direct_buffer.obj(), - fml::jni::VectorToStringArray(env, strings).obj()); + env, view.obj(), direct_buffer.obj(), + fml::jni::VectorToStringArray(env, strings).obj()); } } diff --git a/engine/src/flutter/shell/platform/android/platform_view_android.h b/engine/src/flutter/shell/platform/android/platform_view_android.h index 9976c443f42..b93a9834f3a 100644 --- a/engine/src/flutter/shell/platform/android/platform_view_android.h +++ b/engine/src/flutter/shell/platform/android/platform_view_android.h @@ -75,7 +75,8 @@ class PlatformViewAndroid final : public PlatformView { pending_responses_; // |shell::PlatformView| - void UpdateSemantics(blink::SemanticsNodeUpdates update) override; + void UpdateSemantics(blink::SemanticsNodeUpdates update, + blink::CustomAccessibilityActionUpdates actions) override; // |shell::PlatformView| void HandlePlatformMessage( diff --git a/engine/src/flutter/shell/platform/android/platform_view_android_jni.cc b/engine/src/flutter/shell/platform/android/platform_view_android_jni.cc index 7b0f0d7ae10..0fe2af1c505 100644 --- a/engine/src/flutter/shell/platform/android/platform_view_android_jni.cc +++ b/engine/src/flutter/shell/platform/android/platform_view_android_jni.cc @@ -80,6 +80,15 @@ void FlutterViewUpdateSemantics(JNIEnv* env, FXL_CHECK(CheckException(env)); } +static jmethodID g_update_custom_accessibility_actions_method = nullptr; +void FlutterViewUpdateCustomAccessibilityActions(JNIEnv* env, + jobject obj, + jobject buffer, + jobjectArray strings) { + env->CallVoidMethod(obj, g_update_custom_accessibility_actions_method, buffer, strings); + FXL_CHECK(CheckException(env)); +} + static jmethodID g_on_first_frame_method = nullptr; void FlutterViewOnFirstFrame(JNIEnv* env, jobject obj) { env->CallVoidMethod(obj, g_on_first_frame_method); @@ -648,6 +657,14 @@ bool PlatformViewAndroid::Register(JNIEnv* env) { return false; } + g_update_custom_accessibility_actions_method = + env->GetMethodID(g_flutter_native_view_class->obj(), "updateCustomAccessibilityActions", + "(Ljava/nio/ByteBuffer;[Ljava/lang/String;)V"); + + if (g_update_custom_accessibility_actions_method == nullptr) { + return false; + } + g_on_first_frame_method = env->GetMethodID(g_flutter_native_view_class->obj(), "onFirstFrame", "()V"); diff --git a/engine/src/flutter/shell/platform/android/platform_view_android_jni.h b/engine/src/flutter/shell/platform/android/platform_view_android_jni.h index 65dff978d1a..1a03d1303a7 100644 --- a/engine/src/flutter/shell/platform/android/platform_view_android_jni.h +++ b/engine/src/flutter/shell/platform/android/platform_view_android_jni.h @@ -27,6 +27,11 @@ void FlutterViewUpdateSemantics(JNIEnv* env, jobject buffer, jobjectArray strings); +void FlutterViewUpdateCustomAccessibilityActions(JNIEnv* env, + jobject obj, + jobject buffer, + jobjectArray strings); + void FlutterViewOnFirstFrame(JNIEnv* env, jobject obj); void SurfaceTextureAttachToGLContext(JNIEnv* env, jobject obj, jint textureId); diff --git a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h index ffe99c08032..2ac895d575d 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h +++ b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h @@ -15,6 +15,7 @@ #include "flutter/fml/memory/weak_ptr.h" #include "flutter/fml/platform/darwin/scoped_nsobject.h" #include "flutter/lib/ui/semantics/semantics_node.h" +#include "flutter/lib/ui/semantics/custom_accessibility_action.h" #include "flutter/shell/platform/darwin/ios/framework/Headers/FlutterChannels.h" #include "flutter/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.h" #include "flutter/shell/platform/darwin/ios/framework/Source/FlutterView.h" @@ -80,6 +81,19 @@ class AccessibilityBridge; @end +/** + * An implementation of UIAccessibilityCustomAction which also contains the + * Flutter uid. + */ +@interface FlutterCustomAccessibilityAction : UIAccessibilityCustomAction + +/** + * The uid of the action defined by the flutter application. + */ +@property(nonatomic) int32_t uid; + +@end + /** * The default implementation of `SemanticsObject` for most accessibility elements * in the iOS accessibility tree. @@ -102,8 +116,10 @@ class AccessibilityBridge final { AccessibilityBridge(UIView* view, PlatformViewIOS* platform_view); ~AccessibilityBridge(); - void UpdateSemantics(blink::SemanticsNodeUpdates nodes); + void UpdateSemantics(blink::SemanticsNodeUpdates nodes, blink::CustomAccessibilityActionUpdates actions); void DispatchSemanticsAction(int32_t id, blink::SemanticsAction action); + void DispatchSemanticsAction(int32_t id, blink::SemanticsAction action, std::vector args); + UIView* textInputView(); UIView* view() const { return view_; } @@ -123,6 +139,7 @@ class AccessibilityBridge final { fml::scoped_nsprotocol accessibility_channel_; fml::WeakPtrFactory weak_factory_; int32_t previous_route_id_; + std::unordered_map actions_; std::vector previous_routes_; FXL_DISALLOW_COPY_AND_ASSIGN(AccessibilityBridge); diff --git a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm index f923581e7f0..91bed1ced7b 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm +++ b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm @@ -42,6 +42,11 @@ blink::SemanticsAction GetSemanticsActionForScrollDirection( } // namespace +@implementation FlutterCustomAccessibilityAction + { +} +@end + /** * Represents a semantics object that has children and hence has to be presented to the OS as a * UIAccessibilityContainer. @@ -175,6 +180,20 @@ blink::SemanticsAction GetSemanticsActionForScrollDirection( } } +- (BOOL)onCustomAccessibilityAction:(FlutterCustomAccessibilityAction*)action { + if (![self node].HasAction(blink::SemanticsAction::kCustomAction)) + return NO; + int32_t action_id = action.uid; + std::vector args; + args.push_back(3); // type=int32. + args.push_back(action_id); + args.push_back(action_id >> 8); + args.push_back(action_id >> 16); + args.push_back(action_id >> 24); + [self bridge] ->DispatchSemanticsAction([self uid], blink::SemanticsAction::kCustomAction, args); + return YES; +} + - (NSString*)routeName { // Returns the first non-null and non-empty semantic label of a child // with an NamesRoute flag. Otherwise returns nil. @@ -481,10 +500,14 @@ UIView* AccessibilityBridge::textInputView() { return [platform_view_->GetTextInputPlugin() textInputView]; } -void AccessibilityBridge::UpdateSemantics(blink::SemanticsNodeUpdates nodes) { +void AccessibilityBridge::UpdateSemantics(blink::SemanticsNodeUpdates nodes, + blink::CustomAccessibilityActionUpdates actions) { BOOL layoutChanged = NO; BOOL scrollOccured = NO; - + for (const auto& entry: actions) { + const blink::CustomAccessibilityAction& action = entry.second; + actions_[action.id] = action; + } for (const auto& entry : nodes) { const blink::SemanticsNode& node = entry.second; SemanticsObject* object = GetOrCreateObject(node.id, nodes); @@ -500,6 +523,20 @@ void AccessibilityBridge::UpdateSemantics(blink::SemanticsNodeUpdates nodes) { [newChildren addObject:child]; } object.children = newChildren; + if (node.customAccessibilityActions.size() > 0) { + NSMutableArray* accessibilityCustomActions = + [[[NSMutableArray alloc] init] autorelease]; + for (int32_t action_id : node.customAccessibilityActions) { + blink::CustomAccessibilityAction& action = actions_[action_id]; + NSString* label = @(action.label.data()); + SEL selector = @selector(onCustomAccessibilityAction:); + FlutterCustomAccessibilityAction* customAction = + [[FlutterCustomAccessibilityAction alloc] initWithName:label target:object selector:selector]; + customAction.uid = action_id; + [accessibilityCustomActions addObject:customAction]; + } + object.accessibilityCustomActions = accessibilityCustomActions; + } } SemanticsObject* root = objects_.get()[@(kRootNodeId)]; @@ -560,6 +597,12 @@ void AccessibilityBridge::DispatchSemanticsAction(int32_t uid, blink::SemanticsA platform_view_->DispatchSemanticsAction(uid, action, args); } +void AccessibilityBridge::DispatchSemanticsAction(int32_t uid, + blink::SemanticsAction action, + std::vector args) { + platform_view_->DispatchSemanticsAction(uid, action, args); +} + SemanticsObject* AccessibilityBridge::GetOrCreateObject(int32_t uid, blink::SemanticsNodeUpdates& updates) { SemanticsObject* object = objects_.get()[@(uid)]; diff --git a/engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.h b/engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.h index e7849dda446..3dc29b8b78a 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.h +++ b/engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.h @@ -64,7 +64,8 @@ class PlatformViewIOS final : public PlatformView { fxl::RefPtr message) override; // |shell::PlatformView| - void UpdateSemantics(blink::SemanticsNodeUpdates update) override; + void UpdateSemantics(blink::SemanticsNodeUpdates update, + blink::CustomAccessibilityActionUpdates actions) override; // |shell::PlatformView| std::unique_ptr CreateVSyncWaiter() override; diff --git a/engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.mm b/engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.mm index 7d64bb80cb7..8d77716252f 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.mm +++ b/engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.mm @@ -72,9 +72,10 @@ void PlatformViewIOS::SetSemanticsEnabled(bool enabled) { } // |shell::PlatformView| -void PlatformViewIOS::UpdateSemantics(blink::SemanticsNodeUpdates update) { +void PlatformViewIOS::UpdateSemantics(blink::SemanticsNodeUpdates update, + blink::CustomAccessibilityActionUpdates actions) { if (accessibility_bridge_) { - accessibility_bridge_->UpdateSemantics(std::move(update)); + accessibility_bridge_->UpdateSemantics(std::move(update), std::move(actions)); } }