DEV/flutter_flutter
1
0
Fork 0
mirror of https://github.com/flutter/flutter.git synced 2026-02-20 02:29:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

Custom accessibility (local context) action support for iOS and Android. (flutter/engine#5597)

Browse Source
This commit is contained in:
Jonah Williams 2018-07-11 10:27:50 -07:00 committed by GitHub
parent f227047a5e
commit 5bf14b1e4d
28 changed files with 359 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
engine/src/flutter/lib/ui/BUILD.gn
View File

@ -58,6 +58,8 @@ source_set("ui") {
"semantics/semantics_update.h",
"semantics/semantics_update_builder.cc",
"semantics/semantics_update_builder.h",
"semantics/custom_accessibility_action.cc",
"semantics/custom_accessibility_action.h",
"text/asset_manager_font_provider.cc",
"text/asset_manager_font_provider.h",
"text/font_collection.cc",

24
engine/src/flutter/lib/ui/semantics.dart
View File

@ -26,6 +26,7 @@ class SemanticsAction {
static const int _kPasteIndex = 1 << 14;
static const int _kDidGainAccessibilityFocusIndex = 1 << 15;
static const int _kDidLoseAccessibilityFocusIndex = 1 << 16;
static const int _kCustomAction = 1 << 17;
/// The numerical value for this action.
///
@ -146,6 +147,12 @@ class SemanticsAction {
/// Accessibility focus and input focus can be held by two different nodes!
static const SemanticsAction didLoseAccessibilityFocus = const SemanticsAction._(_kDidLoseAccessibilityFocusIndex);
/// Indicates that the user has invoked a custom accessibility action.
///
/// This handler is added automatically whenever a custom accessibility
/// action is added to a semantics node.
static const SemanticsAction customAction = const SemanticsAction._(_kCustomAction);
/// The possible semantics actions.
///
/// The map's key is the [index] of the action and the value is the action
@ -168,6 +175,7 @@ class SemanticsAction {
_kPasteIndex: paste,
_kDidGainAccessibilityFocusIndex: didGainAccessibilityFocus,
_kDidLoseAccessibilityFocusIndex: didLoseAccessibilityFocus,
_kCustomAction: customAction,
};
@override
@ -207,6 +215,8 @@ class SemanticsAction {
return 'SemanticsAction.didGainAccessibilityFocus';
case _kDidLoseAccessibilityFocusIndex:
return 'SemanticsAction.didLoseAccessibilityFocus';
case _kCustomAction:
return 'SemanticsAction.customAction';
}
return null;
}
@ -498,6 +508,7 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
Float64List transform,
Int32List childrenInTraversalOrder,
Int32List childrenInHitTestOrder,
Int32List customAcccessibilityActions,
}) {
if (transform.length != 16)
throw new ArgumentError('transform argument must have 16 entries.');
@ -523,6 +534,7 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
transform,
childrenInTraversalOrder,
childrenInHitTestOrder,
customAcccessibilityActions,
);
}
void _updateNode(
@ -547,8 +559,20 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
Float64List transform,
Int32List childrenInTraversalOrder,
Int32List childrenInHitTestOrder,
Int32List customAcccessibilityActions,
) native 'SemanticsUpdateBuilder_updateNode';
/// Update the custom accessibility action associated with the given `id`.
///
/// The name of the action exposed to the user is the `label`. The text
/// direction of this label is the same as the global window.
void updateCustomAction({int id, String label}) {
assert(id != null);
assert(label != null && label != '');
_updateCustomAction(id, label);
}
void _updateCustomAction(int id, String label) native 'SemanticsUpdateBuilder_updateAction';
/// Creates a [SemanticsUpdate] object that encapsulates the updates recorded
/// by this object.
///

13
engine/src/flutter/lib/ui/semantics/custom_accessibility_action.cc Normal file
View File

@ -0,0 +1,13 @@
// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "flutter/lib/ui/semantics/custom_accessibility_action.h"
namespace blink {
CustomAccessibilityAction::CustomAccessibilityAction() = default;
CustomAccessibilityAction::~CustomAccessibilityAction() = default;
}

35
engine/src/flutter/lib/ui/semantics/custom_accessibility_action.h Normal file
View File

@ -0,0 +1,35 @@
// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef FLUTTER_LIB_UI_SEMANTICS_CUSTOM_ACCESSIBILITY_ACTION_H_
#define FLUTTER_LIB_UI_SEMANTICS_CUSTOM_ACCESSIBILITY_ACTION_H_
#include "lib/tonic/dart_wrappable.h"
#include "lib/tonic/typed_data/float64_list.h"
#include "lib/tonic/typed_data/int32_list.h"
#include "lib/tonic/dart_library_natives.h"
namespace blink {
/// A custom accessibility action is used to indicate additional semantics
/// actions that a user can perform on a semantics node beyond the
/// preconfigured options.
struct CustomAccessibilityAction {
CustomAccessibilityAction();
~CustomAccessibilityAction();
int32_t id = 0;
std::string label;
};
// Contains custom accessibility actions that need to be updated.
//
// The keys in the map are stable action IDs, and the values contain
// semantic information for the action corresponding to that id.
using CustomAccessibilityActionUpdates = std::unordered_map<int32_t, CustomAccessibilityAction>;
} // namespace blink
#endif //FLUTTER_LIB_UI_SEMANTICS_LOCAL_CONTEXT_ACTION_H_

2
engine/src/flutter/lib/ui/semantics/semantics_node.h
View File

@ -35,6 +35,7 @@ enum class SemanticsAction : int32_t {
kPaste = 1 << 14,
kDidGainAccessibilityFocus = 1 << 15,
kDidLoseAccessibilityFocus = 1 << 16,
kCustomAction = 1 << 17,
};
const int kScrollableSemanticsActions =
@ -87,6 +88,7 @@ struct SemanticsNode {
SkMatrix44 transform = SkMatrix44(SkMatrix44::kIdentity_Constructor);
std::vector<int32_t> childrenInTraversalOrder;
std::vector<int32_t> childrenInHitTestOrder;
std::vector<int32_t> customAccessibilityActions;
};
// Contains semantic nodes that need to be updated.

14
engine/src/flutter/lib/ui/semantics/semantics_update.cc
View File

@ -21,12 +21,14 @@ IMPLEMENT_WRAPPERTYPEINFO(ui, SemanticsUpdate);
DART_BIND_ALL(SemanticsUpdate, FOR_EACH_BINDING)
fxl::RefPtr<SemanticsUpdate> SemanticsUpdate::create(
SemanticsNodeUpdates nodes) {
return fxl::MakeRefCounted<SemanticsUpdate>(std::move(nodes));
SemanticsNodeUpdates nodes,
CustomAccessibilityActionUpdates actions) {
return fxl::MakeRefCounted<SemanticsUpdate>(std::move(nodes), std::move(actions));
}
SemanticsUpdate::SemanticsUpdate(SemanticsNodeUpdates nodes)
: nodes_(std::move(nodes)) {}
SemanticsUpdate::SemanticsUpdate(SemanticsNodeUpdates nodes,
CustomAccessibilityActionUpdates actions)
: nodes_(std::move(nodes)), actions_(std::move(actions)) {}
SemanticsUpdate::~SemanticsUpdate() = default;
@ -34,6 +36,10 @@ SemanticsNodeUpdates SemanticsUpdate::takeNodes() {
return std::move(nodes_);
}
CustomAccessibilityActionUpdates SemanticsUpdate::takeActions() {
return std::move(actions_);
}
void SemanticsUpdate::dispose() {
ClearDartWrapper();
}

10
engine/src/flutter/lib/ui/semantics/semantics_update.h
View File

@ -6,6 +6,7 @@
#define FLUTTER_LIB_UI_SEMANTICS_SEMANTICS_UPDATE_H_
#include "flutter/lib/ui/semantics/semantics_node.h"
#include "flutter/lib/ui/semantics/custom_accessibility_action.h"
#include "lib/tonic/dart_wrappable.h"
namespace tonic {
@ -21,18 +22,23 @@ class SemanticsUpdate : public fxl::RefCountedThreadSafe<SemanticsUpdate>,
public:
~SemanticsUpdate() override;
static fxl::RefPtr<SemanticsUpdate> create(SemanticsNodeUpdates nodes);
static fxl::RefPtr<SemanticsUpdate> create(SemanticsNodeUpdates nodes,
CustomAccessibilityActionUpdates actions);
SemanticsNodeUpdates takeNodes();
CustomAccessibilityActionUpdates takeActions();
void dispose();
static void RegisterNatives(tonic::DartLibraryNatives* natives);
private:
explicit SemanticsUpdate(SemanticsNodeUpdates nodes);
explicit SemanticsUpdate(SemanticsNodeUpdates nodes,
CustomAccessibilityActionUpdates updates);
SemanticsNodeUpdates nodes_;
CustomAccessibilityActionUpdates actions_;
};
} // namespace blink

20
engine/src/flutter/lib/ui/semantics/semantics_update_builder.cc
View File

@ -17,8 +17,9 @@ static void SemanticsUpdateBuilder_constructor(Dart_NativeArguments args) {
IMPLEMENT_WRAPPERTYPEINFO(ui, SemanticsUpdateBuilder);
#define FOR_EACH_BINDING(V) \
V(SemanticsUpdateBuilder, updateNode) \
#define FOR_EACH_BINDING(V) \
V(SemanticsUpdateBuilder, updateNode) \
V(SemanticsUpdateBuilder, updateCustomAction) \
V(SemanticsUpdateBuilder, build)
FOR_EACH_BINDING(DART_NATIVE_CALLBACK)
@ -54,7 +55,8 @@ void SemanticsUpdateBuilder::updateNode(int id,
int textDirection,
const tonic::Float64List& transform,
const tonic::Int32List& childrenInTraversalOrder,
const tonic::Int32List& childrenInHitTestOrder) {
const tonic::Int32List& childrenInHitTestOrder,
const tonic::Int32List& localContextActions) {
SemanticsNode node;
node.id = id;
node.flags = flags;
@ -76,11 +78,21 @@ void SemanticsUpdateBuilder::updateNode(int id,
childrenInTraversalOrder.data(), childrenInTraversalOrder.data() + childrenInTraversalOrder.num_elements());
node.childrenInHitTestOrder = std::vector<int32_t>(
childrenInHitTestOrder.data(), childrenInHitTestOrder.data() + childrenInHitTestOrder.num_elements());
node.customAccessibilityActions = std::vector<int32_t>(
localContextActions.data(), localContextActions.data() + localContextActions.num_elements());
nodes_[id] = node;
}
void SemanticsUpdateBuilder::updateCustomAction(int id,
std::string label) {
CustomAccessibilityAction action;
action.id = id;
action.label = label;
actions_[id] = action;
}
fxl::RefPtr<SemanticsUpdate> SemanticsUpdateBuilder::build() {
return SemanticsUpdate::create(std::move(nodes_));
return SemanticsUpdate::create(std::move(nodes_), std::move(actions_));
}
} // namespace blink

7
engine/src/flutter/lib/ui/semantics/semantics_update_builder.h
View File

@ -45,7 +45,11 @@ class SemanticsUpdateBuilder
int textDirection,
const tonic::Float64List& transform,
const tonic::Int32List& childrenInTraversalOrder,
const tonic::Int32List& childrenInHitTestOrder);
const tonic::Int32List& childrenInHitTestOrder,
const tonic::Int32List& customAccessibilityActions);
void updateCustomAction(int id,
std::string label);
fxl::RefPtr<SemanticsUpdate> build();
@ -55,6 +59,7 @@ class SemanticsUpdateBuilder
explicit SemanticsUpdateBuilder();
SemanticsNodeUpdates nodes_;
CustomAccessibilityActionUpdates actions_;
};
} // namespace blink

2
engine/src/flutter/runtime/runtime_controller.cc
View File

@ -240,7 +240,7 @@ void RuntimeController::Render(Scene* scene) {
void RuntimeController::UpdateSemantics(SemanticsUpdate* update) {
if (window_data_.semantics_enabled) {
client_.UpdateSemantics(update->takeNodes());
client_.UpdateSemantics(update->takeNodes(), update->takeActions());
}
}

4
engine/src/flutter/runtime/runtime_delegate.h
View File

@ -10,6 +10,7 @@
#include "flutter/flow/layers/layer_tree.h"
#include "flutter/lib/ui/semantics/semantics_node.h"
#include "flutter/lib/ui/semantics/custom_accessibility_action.h"
#include "flutter/lib/ui/text/font_collection.h"
#include "flutter/lib/ui/window/platform_message.h"
#include "third_party/dart/runtime/include/dart_api.h"
@ -24,7 +25,8 @@ class RuntimeDelegate {
virtual void Render(std::unique_ptr<flow::LayerTree> layer_tree) = 0;
virtual void UpdateSemantics(blink::SemanticsNodeUpdates update) = 0;
virtual void UpdateSemantics(blink::SemanticsNodeUpdates update,
blink::CustomAccessibilityActionUpdates actions) = 0;
virtual void HandlePlatformMessage(fxl::RefPtr<PlatformMessage> message) = 0;

5
engine/src/flutter/shell/common/engine.cc
View File

@ -364,8 +364,9 @@ void Engine::Render(std::unique_ptr<flow::LayerTree> layer_tree) {
animator_->Render(std::move(layer_tree));
}
void Engine::UpdateSemantics(blink::SemanticsNodeUpdates update) {
delegate_.OnEngineUpdateSemantics(*this, std::move(update));
void Engine::UpdateSemantics(blink::SemanticsNodeUpdates update,
blink::CustomAccessibilityActionUpdates actions) {
delegate_.OnEngineUpdateSemantics(*this, std::move(update), std::move(actions));
}
void Engine::HandlePlatformMessage(

7
engine/src/flutter/shell/common/engine.h
View File

@ -11,6 +11,7 @@
#include "flutter/assets/asset_manager.h"
#include "flutter/common/task_runners.h"
#include "flutter/lib/ui/semantics/semantics_node.h"
#include "flutter/lib/ui/semantics/custom_accessibility_action.h"
#include "flutter/lib/ui/text/font_collection.h"
#include "flutter/lib/ui/window/platform_message.h"
#include "flutter/lib/ui/window/viewport_metrics.h"
@ -32,7 +33,8 @@ class Engine final : public blink::RuntimeDelegate {
public:
virtual void OnEngineUpdateSemantics(
const Engine& engine,
blink::SemanticsNodeUpdates update) = 0;
blink::SemanticsNodeUpdates update,
blink::CustomAccessibilityActionUpdates actions) = 0;
virtual void OnEngineHandlePlatformMessage(
const Engine& engine,
@ -123,7 +125,8 @@ class Engine final : public blink::RuntimeDelegate {
void Render(std::unique_ptr<flow::LayerTree> layer_tree) override;
// |blink::RuntimeDelegate|
void UpdateSemantics(blink::SemanticsNodeUpdates update) override;
void UpdateSemantics(blink::SemanticsNodeUpdates update,
blink::CustomAccessibilityActionUpdates actions) override;
// |blink::RuntimeDelegate|
void HandlePlatformMessage(

3
engine/src/flutter/shell/common/platform_view.cc
View File

@ -74,7 +74,8 @@ fml::WeakPtr<PlatformView> PlatformView::GetWeakPtr() const {
return weak_factory_.GetWeakPtr();
}
void PlatformView::UpdateSemantics(blink::SemanticsNodeUpdates update) {}
void PlatformView::UpdateSemantics(blink::SemanticsNodeUpdates update,
blink::CustomAccessibilityActionUpdates actions) {}
void PlatformView::HandlePlatformMessage(
fxl::RefPtr<blink::PlatformMessage> message) {

4
engine/src/flutter/shell/common/platform_view.h
View File

@ -11,6 +11,7 @@
#include "flutter/flow/texture.h"
#include "flutter/fml/memory/weak_ptr.h"
#include "flutter/lib/ui/semantics/semantics_node.h"
#include "flutter/lib/ui/semantics/custom_accessibility_action.h"
#include "flutter/lib/ui/window/platform_message.h"
#include "flutter/lib/ui/window/pointer_data_packet.h"
#include "flutter/lib/ui/window/viewport_metrics.h"
@ -95,7 +96,8 @@ class PlatformView {
fml::WeakPtr<PlatformView> GetWeakPtr() const;
virtual void UpdateSemantics(blink::SemanticsNodeUpdates update);
virtual void UpdateSemantics(blink::SemanticsNodeUpdates updates,
blink::CustomAccessibilityActionUpdates actions);
virtual void HandlePlatformMessage(
fxl::RefPtr<blink::PlatformMessage> message);

7
engine/src/flutter/shell/common/shell.cc
View File

@ -707,14 +707,15 @@ void Shell::OnAnimatorDrawLastLayerTree(const Animator& animator) {
// |shell::Engine::Delegate|
void Shell::OnEngineUpdateSemantics(const Engine& engine,
blink::SemanticsNodeUpdates update) {
blink::SemanticsNodeUpdates update,
blink::CustomAccessibilityActionUpdates actions) {
FXL_DCHECK(is_setup_);
FXL_DCHECK(task_runners_.GetUITaskRunner()->RunsTasksOnCurrentThread());
task_runners_.GetPlatformTaskRunner()->PostTask(
[view = platform_view_->GetWeakPtr(), update = std::move(update)] {
[view = platform_view_->GetWeakPtr(), update = std::move(update), actions = std::move(actions)] {
if (view) {
view->UpdateSemantics(std::move(update));
view->UpdateSemantics(std::move(update), std::move(actions));
}
});
}

4
engine/src/flutter/shell/common/shell.h
View File

@ -15,6 +15,7 @@
#include "flutter/fml/memory/weak_ptr.h"
#include "flutter/fml/thread.h"
#include "flutter/lib/ui/semantics/semantics_node.h"
#include "flutter/lib/ui/semantics/custom_accessibility_action.h"
#include "flutter/lib/ui/window/platform_message.h"
#include "flutter/runtime/service_protocol.h"
#include "flutter/shell/common/animator.h"
@ -183,7 +184,8 @@ class Shell final : public PlatformView::Delegate,
// |shell::Engine::Delegate|
void OnEngineUpdateSemantics(const Engine& engine,
blink::SemanticsNodeUpdates update) override;
blink::SemanticsNodeUpdates update,
blink::CustomAccessibilityActionUpdates actions) override;
// |shell::Engine::Delegate|
void OnEngineHandlePlatformMessage(

72
engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java
View File

@ -32,6 +32,7 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
private static final int ROOT_NODE_ID = 0;
private Map<Integer, SemanticsObject> mObjects;
private Map<Integer, CustomAccessibilityAction> mCustomAccessibilityActions;
private final FlutterView mOwner;
private boolean mAccessibilityEnabled = false;
private SemanticsObject mA11yFocusedObject;
@ -59,7 +60,8 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
CUT(1 << 13),
PASTE(1 << 14),
DID_GAIN_ACCESSIBILITY_FOCUS(1 << 15),
DID_LOSE_ACCESSIBILITY_FOCUS(1 << 16);
DID_LOSE_ACCESSIBILITY_FOCUS(1 << 16),
CUSTOM_ACTION(1 << 17);
Action(int value) {
this.value = value;
@ -95,6 +97,7 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
assert owner != null;
mOwner = owner;
mObjects = new HashMap<Integer, SemanticsObject>();
mCustomAccessibilityActions = new HashMap<Integer, CustomAccessibilityAction>();
previousRoutes = new ArrayList<>();
mFlutterAccessibilityChannel = new BasicMessageChannel<>(owner, "flutter/accessibility",
StandardMessageCodec.INSTANCE);
@ -250,6 +253,15 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
result.addAction(AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS);
}
// Actions on the local context menu
if (Build.VERSION.SDK_INT >= 21) {
if (object.customAccessibilityAction != null) {
for (CustomAccessibilityAction action : object.customAccessibilityAction) {
result.addAction(new AccessibilityNodeInfo.AccessibilityAction(action.resourceId, action.label));
}
}
}
if (object.childrenInTraversalOrder != null) {
for (SemanticsObject child : object.childrenInTraversalOrder) {
if (!child.hasFlag(Flag.IS_HIDDEN)) {
@ -381,6 +393,14 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
mOwner.dispatchSemanticsAction(virtualViewId, Action.PASTE);
return true;
}
default:
// might be a custom accessibility action.
final int flutterId = action - firstResourceId;
CustomAccessibilityAction contextAction = mCustomAccessibilityActions.get(flutterId);
if (contextAction != null) {
mOwner.dispatchSemanticsAction(virtualViewId, Action.CUSTOM_ACTION, contextAction.id);
return true;
}
}
return false;
}
@ -440,6 +460,17 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
return object;
}
private CustomAccessibilityAction getOrCreateAction(int id) {
CustomAccessibilityAction action = mCustomAccessibilityActions.get(id);
if (action == null) {
action = new CustomAccessibilityAction();
action.id = id;
action.resourceId = id + firstResourceId;
mCustomAccessibilityActions.put(id, action);
}
return action;
}
void handleTouchExplorationExit() {
if (mHoveredObject != null) {
sendAccessibilityEvent(mHoveredObject.id, AccessibilityEvent.TYPE_VIEW_HOVER_EXIT);
@ -464,6 +495,16 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
}
}
void updateCustomAccessibilityActions(ByteBuffer buffer, String[] strings) {
ArrayList<CustomAccessibilityAction> updatedActions = new ArrayList<CustomAccessibilityAction>();
while (buffer.hasRemaining()) {
int id = buffer.getInt();
CustomAccessibilityAction action = getOrCreateAction(id);
int stringIndex = buffer.getInt();
action.label = stringIndex == -1 ? null : strings[stringIndex];
}
}
void updateSemantics(ByteBuffer buffer, String[] strings) {
ArrayList<SemanticsObject> updated = new ArrayList<SemanticsObject>();
while (buffer.hasRemaining()) {
@ -732,6 +773,20 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
}
}
private class CustomAccessibilityAction {
CustomAccessibilityAction() {}
/// Resource id is the id of the custom action plus a minimum value so that the identifier
/// does not collide with existing Android accessibility actions.
int resourceId = -1;
int id = -1;
/// The label is the user presented value which is displayed in the local context menu.
String label;
}
/// Value is derived from ACTION_TYPE_MASK in AccessibilityNodeInfo.java
static int firstResourceId = 267386881;
private class SemanticsObject {
SemanticsObject() { }
@ -770,6 +825,7 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
SemanticsObject parent;
List<SemanticsObject> childrenInTraversalOrder;
List<SemanticsObject> childrenInHitTestOrder;
List<CustomAccessibilityAction> customAccessibilityAction;
private boolean inverseTransformDirty = true;
private float[] inverseTransform;
@ -888,6 +944,20 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
childrenInHitTestOrder.add(child);
}
}
final int actionCount = buffer.getInt();
if (actionCount == 0) {
customAccessibilityAction = null;
} else {
if (customAccessibilityAction == null)
customAccessibilityAction = new ArrayList<CustomAccessibilityAction>(actionCount);
else
customAccessibilityAction.clear();
for (int i = 0; i < actionCount; i++) {
CustomAccessibilityAction action = getOrCreateAction(buffer.getInt());
customAccessibilityAction.add(action);
}
}
}
private void ensureInverseTransform() {

7
engine/src/flutter/shell/platform/android/io/flutter/view/FlutterNativeView.java
View File

@ -183,6 +183,13 @@ public class FlutterNativeView implements BinaryMessenger {
mFlutterView.updateSemantics(buffer, strings);
}
// Called by native to update the custom accessibility actions.
private void updateCustomAccessibilityActions(ByteBuffer buffer, String[] strings) {
if (mFlutterView == null)
return;
mFlutterView.updateCustomAccessibilityActions(buffer, strings);
}
// Called by native to notify first Flutter frame rendered.
private void onFirstFrame() {
if (mFlutterView == null)

11
engine/src/flutter/shell/platform/android/io/flutter/view/FlutterView.java
View File

@ -712,6 +712,17 @@ public class FlutterView extends SurfaceView
}
}
public void updateCustomAccessibilityActions(ByteBuffer buffer, String[] strings) {
try {
if (mAccessibilityNodeProvider != null) {
buffer.order(ByteOrder.LITTLE_ENDIAN);
mAccessibilityNodeProvider.updateCustomAccessibilityActions(buffer, strings);
}
} catch (Exception ex) {
Log.e(TAG, "Uncaught exception while updating local context actions", ex);
}
}
// Called by native to notify first Flutter frame rendered.
public void onFirstFrame() {
// Allow listeners to remove themselves when they are called.

42
engine/src/flutter/shell/platform/android/platform_view_android.cc
View File

@ -178,9 +178,11 @@ void PlatformViewAndroid::DispatchSemanticsAction(JNIEnv* env,
}
// |shell::PlatformView|
void PlatformViewAndroid::UpdateSemantics(blink::SemanticsNodeUpdates update) {
constexpr size_t kBytesPerNode = 35 * sizeof(int32_t);
void PlatformViewAndroid::UpdateSemantics(blink::SemanticsNodeUpdates update,
blink::CustomAccessibilityActionUpdates actions) {
constexpr size_t kBytesPerNode = 36 * sizeof(int32_t);
constexpr size_t kBytesPerChild = sizeof(int32_t);
constexpr size_t kBytesPerAction = 2 * sizeof(int32_t);
JNIEnv* env = fml::jni::AttachCurrentThread();
{
@ -194,6 +196,7 @@ void PlatformViewAndroid::UpdateSemantics(blink::SemanticsNodeUpdates update) {
num_bytes +=
value.second.childrenInTraversalOrder.size() * kBytesPerChild;
num_bytes += value.second.childrenInHitTestOrder.size() * kBytesPerChild;
num_bytes += value.second.customAccessibilityActions.size() * kBytesPerChild;
}
std::vector<uint8_t> buffer(num_bytes);
@ -259,14 +262,45 @@ void PlatformViewAndroid::UpdateSemantics(blink::SemanticsNodeUpdates update) {
for (int32_t child : node.childrenInHitTestOrder)
buffer_int32[position++] = child;
buffer_int32[position++] = node.customAccessibilityActions.size();
for (int32_t child : node.customAccessibilityActions)
buffer_int32[position++] = child;
}
// custom accessibility actions.
size_t num_action_bytes = actions.size() * kBytesPerAction;
std::vector<uint8_t> actions_buffer(num_action_bytes);
int32_t* actions_buffer_int32 = reinterpret_cast<int32_t*>(&actions_buffer[0]);
std::vector<std::string> action_strings;
size_t actions_position = 0;
for (const auto& value : actions) {
// If you edit this code, make sure you update kBytesPerAction
// to match the number of values you are
// sending.
const blink::CustomAccessibilityAction& action = value.second;
actions_buffer_int32[actions_position++] = action.id;
if (action.label.empty()) {
actions_buffer_int32[actions_position++] = -1;
} else {
actions_buffer_int32[actions_position++] = action_strings.size();
action_strings.push_back(action.label);
}
}
fml::jni::ScopedJavaLocalRef<jobject> direct_actions_buffer(
env, env->NewDirectByteBuffer(actions_buffer.data(), actions_buffer.size()));
fml::jni::ScopedJavaLocalRef<jobject> direct_buffer(
env, env->NewDirectByteBuffer(buffer.data(), buffer.size()));
FlutterViewUpdateCustomAccessibilityActions(
env, view.obj(), direct_actions_buffer.obj(),
fml::jni::VectorToStringArray(env, action_strings).obj());
FlutterViewUpdateSemantics(
env, view.obj(), direct_buffer.obj(),
fml::jni::VectorToStringArray(env, strings).obj());
env, view.obj(), direct_buffer.obj(),
fml::jni::VectorToStringArray(env, strings).obj());
}
}

3
engine/src/flutter/shell/platform/android/platform_view_android.h
View File

@ -75,7 +75,8 @@ class PlatformViewAndroid final : public PlatformView {
pending_responses_;
// |shell::PlatformView|
void UpdateSemantics(blink::SemanticsNodeUpdates update) override;
void UpdateSemantics(blink::SemanticsNodeUpdates update,
blink::CustomAccessibilityActionUpdates actions) override;
// |shell::PlatformView|
void HandlePlatformMessage(

17
engine/src/flutter/shell/platform/android/platform_view_android_jni.cc
View File

@ -80,6 +80,15 @@ void FlutterViewUpdateSemantics(JNIEnv* env,
FXL_CHECK(CheckException(env));
}
static jmethodID g_update_custom_accessibility_actions_method = nullptr;
void FlutterViewUpdateCustomAccessibilityActions(JNIEnv* env,
jobject obj,
jobject buffer,
jobjectArray strings) {
env->CallVoidMethod(obj, g_update_custom_accessibility_actions_method, buffer, strings);
FXL_CHECK(CheckException(env));
}
static jmethodID g_on_first_frame_method = nullptr;
void FlutterViewOnFirstFrame(JNIEnv* env, jobject obj) {
env->CallVoidMethod(obj, g_on_first_frame_method);
@ -648,6 +657,14 @@ bool PlatformViewAndroid::Register(JNIEnv* env) {
return false;
}
g_update_custom_accessibility_actions_method =
env->GetMethodID(g_flutter_native_view_class->obj(), "updateCustomAccessibilityActions",
"(Ljava/nio/ByteBuffer;[Ljava/lang/String;)V");
if (g_update_custom_accessibility_actions_method == nullptr) {
return false;
}
g_on_first_frame_method = env->GetMethodID(g_flutter_native_view_class->obj(),
"onFirstFrame", "()V");

5
engine/src/flutter/shell/platform/android/platform_view_android_jni.h
View File

@ -27,6 +27,11 @@ void FlutterViewUpdateSemantics(JNIEnv* env,
jobject buffer,
jobjectArray strings);
void FlutterViewUpdateCustomAccessibilityActions(JNIEnv* env,
jobject obj,
jobject buffer,
jobjectArray strings);
void FlutterViewOnFirstFrame(JNIEnv* env, jobject obj);
void SurfaceTextureAttachToGLContext(JNIEnv* env, jobject obj, jint textureId);

19
engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h
View File

@ -15,6 +15,7 @@
#include "flutter/fml/memory/weak_ptr.h"
#include "flutter/fml/platform/darwin/scoped_nsobject.h"
#include "flutter/lib/ui/semantics/semantics_node.h"
#include "flutter/lib/ui/semantics/custom_accessibility_action.h"
#include "flutter/shell/platform/darwin/ios/framework/Headers/FlutterChannels.h"
#include "flutter/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.h"
#include "flutter/shell/platform/darwin/ios/framework/Source/FlutterView.h"
@ -80,6 +81,19 @@ class AccessibilityBridge;
@end
/**
* An implementation of UIAccessibilityCustomAction which also contains the
* Flutter uid.
*/
@interface FlutterCustomAccessibilityAction : UIAccessibilityCustomAction
/**
* The uid of the action defined by the flutter application.
*/
@property(nonatomic) int32_t uid;
@end
/**
* The default implementation of `SemanticsObject` for most accessibility elements
* in the iOS accessibility tree.
@ -102,8 +116,10 @@ class AccessibilityBridge final {
AccessibilityBridge(UIView* view, PlatformViewIOS* platform_view);
~AccessibilityBridge();
void UpdateSemantics(blink::SemanticsNodeUpdates nodes);
void UpdateSemantics(blink::SemanticsNodeUpdates nodes, blink::CustomAccessibilityActionUpdates actions);
void DispatchSemanticsAction(int32_t id, blink::SemanticsAction action);
void DispatchSemanticsAction(int32_t id, blink::SemanticsAction action, std::vector<uint8_t> args);
UIView<UITextInput>* textInputView();
UIView* view() const { return view_; }
@ -123,6 +139,7 @@ class AccessibilityBridge final {
fml::scoped_nsprotocol<FlutterBasicMessageChannel*> accessibility_channel_;
fml::WeakPtrFactory<AccessibilityBridge> weak_factory_;
int32_t previous_route_id_;
std::unordered_map<int32_t, blink::CustomAccessibilityAction> actions_;
std::vector<int32_t> previous_routes_;
FXL_DISALLOW_COPY_AND_ASSIGN(AccessibilityBridge);

47
engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm
View File

@ -42,6 +42,11 @@ blink::SemanticsAction GetSemanticsActionForScrollDirection(
} // namespace
@implementation FlutterCustomAccessibilityAction
{
}
@end
/**
* Represents a semantics object that has children and hence has to be presented to the OS as a
* UIAccessibilityContainer.
@ -175,6 +180,20 @@ blink::SemanticsAction GetSemanticsActionForScrollDirection(
}
}
- (BOOL)onCustomAccessibilityAction:(FlutterCustomAccessibilityAction*)action {
if (![self node].HasAction(blink::SemanticsAction::kCustomAction))
return NO;
int32_t action_id = action.uid;
std::vector<uint8_t> args;
args.push_back(3); // type=int32.
args.push_back(action_id);
args.push_back(action_id >> 8);
args.push_back(action_id >> 16);
args.push_back(action_id >> 24);
[self bridge] ->DispatchSemanticsAction([self uid], blink::SemanticsAction::kCustomAction, args);
return YES;
}
- (NSString*)routeName {
// Returns the first non-null and non-empty semantic label of a child
// with an NamesRoute flag. Otherwise returns nil.
@ -481,10 +500,14 @@ UIView<UITextInput>* AccessibilityBridge::textInputView() {
return [platform_view_->GetTextInputPlugin() textInputView];
}
void AccessibilityBridge::UpdateSemantics(blink::SemanticsNodeUpdates nodes) {
void AccessibilityBridge::UpdateSemantics(blink::SemanticsNodeUpdates nodes,
blink::CustomAccessibilityActionUpdates actions) {
BOOL layoutChanged = NO;
BOOL scrollOccured = NO;
for (const auto& entry: actions) {
const blink::CustomAccessibilityAction& action = entry.second;
actions_[action.id] = action;
}
for (const auto& entry : nodes) {
const blink::SemanticsNode& node = entry.second;
SemanticsObject* object = GetOrCreateObject(node.id, nodes);
@ -500,6 +523,20 @@ void AccessibilityBridge::UpdateSemantics(blink::SemanticsNodeUpdates nodes) {
[newChildren addObject:child];
}
object.children = newChildren;
if (node.customAccessibilityActions.size() > 0) {
NSMutableArray<FlutterCustomAccessibilityAction*>* accessibilityCustomActions =
[[[NSMutableArray alloc] init] autorelease];
for (int32_t action_id : node.customAccessibilityActions) {
blink::CustomAccessibilityAction& action = actions_[action_id];
NSString* label = @(action.label.data());
SEL selector = @selector(onCustomAccessibilityAction:);
FlutterCustomAccessibilityAction* customAction =
[[FlutterCustomAccessibilityAction alloc] initWithName:label target:object selector:selector];
customAction.uid = action_id;
[accessibilityCustomActions addObject:customAction];
}
object.accessibilityCustomActions = accessibilityCustomActions;
}
}
SemanticsObject* root = objects_.get()[@(kRootNodeId)];
@ -560,6 +597,12 @@ void AccessibilityBridge::DispatchSemanticsAction(int32_t uid, blink::SemanticsA
platform_view_->DispatchSemanticsAction(uid, action, args);
}
void AccessibilityBridge::DispatchSemanticsAction(int32_t uid,
blink::SemanticsAction action,
std::vector<uint8_t> args) {
platform_view_->DispatchSemanticsAction(uid, action, args);
}
SemanticsObject* AccessibilityBridge::GetOrCreateObject(int32_t uid,
blink::SemanticsNodeUpdates& updates) {
SemanticsObject* object = objects_.get()[@(uid)];

3
engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.h
View File

@ -64,7 +64,8 @@ class PlatformViewIOS final : public PlatformView {
fxl::RefPtr<blink::PlatformMessage> message) override;
// |shell::PlatformView|
void UpdateSemantics(blink::SemanticsNodeUpdates update) override;
void UpdateSemantics(blink::SemanticsNodeUpdates update,
blink::CustomAccessibilityActionUpdates actions) override;
// |shell::PlatformView|
std::unique_ptr<VsyncWaiter> CreateVSyncWaiter() override;

5
engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.mm
View File

@ -72,9 +72,10 @@ void PlatformViewIOS::SetSemanticsEnabled(bool enabled) {
}
// |shell::PlatformView|
void PlatformViewIOS::UpdateSemantics(blink::SemanticsNodeUpdates update) {
void PlatformViewIOS::UpdateSemantics(blink::SemanticsNodeUpdates update,
blink::CustomAccessibilityActionUpdates actions) {
if (accessibility_bridge_) {
accessibility_bridge_->UpdateSemantics(std::move(update));
accessibility_bridge_->UpdateSemantics(std::move(update), std::move(actions));
}
}