DEV/flutter_flutter
1
0
Fork 0
mirror of https://github.com/flutter/flutter.git synced 2026-02-20 02:29:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

Harmonize Android and iOS accessibility bridges (#2777)

Browse Source 
These classes now use the same terminology and work in the same way.
Also, change semantics.mojom to use an enumeration of actions instead of
having a separate method per action. This will hopefully scale better.
This commit is contained in:
Adam Barth 2016-06-30 16:49:04 -07:00 committed by GitHub
parent 2eb81e33bc
commit 71ce354a44
5 changed files with 360 additions and 295 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
sky/services/semantics/semantics.mojom
View File

@ -5,6 +5,17 @@
[DartPackage="sky_services"]
module semantics;
enum SemanticAction {
TAP,
LONG_PRESS,
SCROLL_LEFT,
SCROLL_RIGHT,
SCROLL_UP,
SCROLL_DOWN,
INCREASE,
DECREASE,
};
struct SemanticsNode {
uint32 id;
@ -12,19 +23,17 @@ struct SemanticsNode {
SemanticFlags? flags;
SemanticStrings? strings;
SemanticGeometry? geometry;
// TODO(abarth): Switch to array<SemanticAction> once that works.
// See https://github.com/domokit/mojo/issues/799
array<int32>? actions;
array<SemanticsNode>? children;
};
struct SemanticFlags {
// This is intended to just be booleans, so that it can be extended
// over time yet still be packed tightly.
bool canBeTapped = false;
bool canBeLongPressed = false;
bool canBeScrolledHorizontally = false;
bool canBeScrolledVertically = false;
bool hasCheckedState = false; // whether isChecked is relevant
bool isChecked = false;
bool isAdjustable = false;
bool has_checked_state = false; // whether is_checked is relevant
bool is_checked = false;
};
struct SemanticStrings {
@ -56,7 +65,7 @@ struct SemanticGeometry {
};
interface SemanticsListener {
// The OS side, invoked from the app.
// The engine side, invoked from the app.
UpdateSemanticsTree(array<SemanticsNode> nodes);
};
@ -64,12 +73,5 @@ interface SemanticsListener {
interface SemanticsServer {
// The app side, invoked from the engine.
AddSemanticsListener(SemanticsListener listener);
Tap(uint32 nodeID);
LongPress(uint32 nodeID);
ScrollLeft(uint32 nodeID);
ScrollRight(uint32 nodeID);
ScrollUp(uint32 nodeID);
ScrollDown(uint32 nodeID);
AdjustIncrease(uint32 nodeID);
AdjustDecrease(uint32 nodeID);
PerformAction(uint32 nodeID, SemanticAction action);
};

295
sky/shell/platform/android/io/flutter/view/AccessibilityBridge.java
View File

@ -14,6 +14,7 @@ import android.view.accessibility.AccessibilityNodeInfo;
import android.view.accessibility.AccessibilityNodeProvider;
import org.chromium.mojo.system.MojoException;
import org.chromium.mojom.semantics.SemanticAction;
import org.chromium.mojom.semantics.SemanticsListener;
import org.chromium.mojom.semantics.SemanticsNode;
import org.chromium.mojom.semantics.SemanticsServer;
@ -21,22 +22,24 @@ import org.chromium.mojom.sky.ViewportMetrics;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
class AccessibilityBridge extends AccessibilityNodeProvider implements SemanticsListener {
private Map<Integer, PersistentAccessibilityNode> mTreeNodes;
private Map<Integer, SemanticObject> mObjects;
private FlutterView mOwner;
private SemanticsServer.Proxy mSemanticsServer;
private boolean mAccessibilityEnabled = false;
private PersistentAccessibilityNode mFocusedNode;
private PersistentAccessibilityNode mHoveredNode;
private SemanticObject mFocusedObject;
private SemanticObject mHoveredObject;
AccessibilityBridge(FlutterView owner, SemanticsServer.Proxy semanticsServer) {
assert owner != null;
assert semanticsServer != null;
mOwner = owner;
mTreeNodes = new HashMap<Integer, PersistentAccessibilityNode>();
mObjects = new HashMap<Integer, SemanticObject>();
mSemanticsServer = semanticsServer;
mSemanticsServer.addSemanticsListener(this);
}
@ -47,17 +50,16 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
@Override
public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId) {
if (virtualViewId == View.NO_ID) {
AccessibilityNodeInfo result = AccessibilityNodeInfo.obtain(mOwner);
mOwner.onInitializeAccessibilityNodeInfo(result);
if (mTreeNodes.containsKey(0))
if (mObjects.containsKey(0))
result.addChild(mOwner, 0);
return result;
}
PersistentAccessibilityNode node = mTreeNodes.get(virtualViewId);
if (node == null)
SemanticObject object = mObjects.get(virtualViewId);
if (object == null)
return null;
AccessibilityNodeInfo result = AccessibilityNodeInfo.obtain(mOwner, virtualViewId);
@ -65,17 +67,17 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
result.setClassName("Flutter"); // Prettier than the more conventional node.getClass().getName()
result.setSource(mOwner, virtualViewId);
if (node.parent != null) {
assert node.id > 0;
result.setParent(mOwner, node.parent.id);
if (object.parent != null) {
assert object.id > 0;
result.setParent(mOwner, object.parent.id);
} else {
assert node.id == 0;
assert object.id == 0;
result.setParent(mOwner);
}
Rect bounds = node.getGlobalRect();
if (node.parent != null) {
Rect parentBounds = node.parent.getGlobalRect();
Rect bounds = object.getGlobalRect();
if (object.parent != null) {
Rect parentBounds = object.parent.getGlobalRect();
Rect boundsInParent = new Rect(bounds);
boundsInParent.offset(-parentBounds.left, -parentBounds.top);
result.setBoundsInParent(boundsInParent);
@ -86,15 +88,15 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
result.setVisibleToUser(true);
result.setEnabled(true); // TODO(ianh): Expose disabled subtrees
if (node.canBeTapped) {
if (object.canBeTapped) {
result.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_CLICK);
result.setClickable(true);
}
if (node.canBeLongPressed) {
if (object.canBeLongPressed) {
result.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_LONG_CLICK);
result.setLongClickable(true);
}
if (node.canBeScrolledHorizontally || node.canBeScrolledVertically) {
if (object.canBeScrolledHorizontally || object.canBeScrolledVertically) {
// TODO(ianh): Once we're on SDK v23+, call addAction to
// expose AccessibilityAction.ACTION_SCROLL_LEFT, _RIGHT,
// _UP, and _DOWN when appropriate.
@ -104,9 +106,9 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
result.setScrollable(true);
}
result.setCheckable(node.hasCheckedState);
result.setChecked(node.isChecked);
result.setText(node.label);
result.setCheckable(object.hasCheckedState);
result.setChecked(object.isChecked);
result.setText(object.label);
// TODO(ianh): use setTraversalBefore/setTraversalAfter to set
// the relative order of the views. For each set of siblings,
@ -115,14 +117,14 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
// width, and finally by list order.
// Accessibility Focus
if (mFocusedNode != null && mFocusedNode.id == virtualViewId) {
if (mFocusedObject != null && mFocusedObject.id == virtualViewId) {
result.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_CLEAR_ACCESSIBILITY_FOCUS);
} else {
result.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_ACCESSIBILITY_FOCUS);
}
if (node.children != null) {
for (PersistentAccessibilityNode child : node.children) {
if (object.children != null) {
for (SemanticObject child : object.children) {
result.addChild(mOwner, child.id);
}
}
@ -132,35 +134,36 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
@Override
public boolean performAction(int virtualViewId, int action, Bundle arguments) {
PersistentAccessibilityNode node = mTreeNodes.get(virtualViewId);
if (node == null)
SemanticObject object = mObjects.get(virtualViewId);
if (object == null) {
return false;
}
switch (action) {
case AccessibilityNodeInfo.ACTION_CLICK: {
mSemanticsServer.tap(virtualViewId);
mSemanticsServer.performAction(virtualViewId, SemanticAction.TAP);
return true;
}
case AccessibilityNodeInfo.ACTION_LONG_CLICK: {
mSemanticsServer.longPress(virtualViewId);
mSemanticsServer.performAction(virtualViewId, SemanticAction.LONG_PRESS);
return true;
}
case AccessibilityNodeInfo.ACTION_SCROLL_BACKWARD: {
if (node.canBeScrolledVertically) {
mSemanticsServer.scrollUp(virtualViewId);
} else if (node.canBeScrolledHorizontally) {
if (object.canBeScrolledVertically) {
mSemanticsServer.performAction(virtualViewId, SemanticAction.SCROLL_UP);
} else if (object.canBeScrolledHorizontally) {
// TODO(ianh): bidi support
mSemanticsServer.scrollLeft(virtualViewId);
mSemanticsServer.performAction(virtualViewId, SemanticAction.SCROLL_LEFT);
} else {
return false;
}
return true;
}
case AccessibilityNodeInfo.ACTION_SCROLL_FORWARD: {
if (node.canBeScrolledVertically) {
mSemanticsServer.scrollDown(virtualViewId);
} else if (node.canBeScrolledHorizontally) {
if (object.canBeScrolledVertically) {
mSemanticsServer.performAction(virtualViewId, SemanticAction.SCROLL_DOWN);
} else if (object.canBeScrolledHorizontally) {
// TODO(ianh): bidi support
mSemanticsServer.scrollRight(virtualViewId);
mSemanticsServer.performAction(virtualViewId, SemanticAction.SCROLL_RIGHT);
} else {
return false;
}
@ -168,58 +171,107 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
}
case AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS: {
sendAccessibilityEvent(virtualViewId, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED);
mFocusedNode = null;
mFocusedObject = null;
return true;
}
case AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS: {
sendAccessibilityEvent(virtualViewId, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED);
if (mFocusedNode == null) {
if (mFocusedObject == null) {
// When Android focuses a node, it doesn't invalidate the view.
// (It does when it sends ACTION_CLEAR_ACCESSIBILITY_FOCUS, so
// we only have to worry about this when the focused node is null.)
mOwner.invalidate();
}
mFocusedNode = node;
mFocusedObject = object;
return true;
}
}
// TODO(ianh): Implement left/right/up/down scrolling
return false;
}
// TODO(ianh): implement findAccessibilityNodeInfosByText()
// TODO(ianh): implement findFocus()
private SemanticObject getRootObject() {
return mObjects.get(0);
}
void handleTouchExplorationExit() {
if (mHoveredNode != null) {
sendAccessibilityEvent(mHoveredNode.id, AccessibilityEvent.TYPE_VIEW_HOVER_EXIT);
mHoveredNode = null;
if (mHoveredObject != null) {
sendAccessibilityEvent(mHoveredObject.id, AccessibilityEvent.TYPE_VIEW_HOVER_EXIT);
mHoveredObject = null;
}
}
void handleTouchExploration(float x, float y) {
if (mTreeNodes.isEmpty())
if (mObjects.isEmpty()) {
return;
assert mTreeNodes.containsKey(0);
PersistentAccessibilityNode newNode = mTreeNodes.get(0).hitTest(Math.round(x), Math.round(y));
if (newNode != mHoveredNode) {
}
assert mObjects.containsKey(0);
SemanticObject newObject = getRootObject().hitTest(Math.round(x), Math.round(y));
if (newObject != mHoveredObject) {
// sending ENTER before EXIT is how Android wants it
if (newNode != null) {
sendAccessibilityEvent(newNode.id, AccessibilityEvent.TYPE_VIEW_HOVER_ENTER);
if (newObject != null) {
sendAccessibilityEvent(newObject.id, AccessibilityEvent.TYPE_VIEW_HOVER_ENTER);
}
if (mHoveredNode != null) {
sendAccessibilityEvent(mHoveredNode.id, AccessibilityEvent.TYPE_VIEW_HOVER_EXIT);
if (mHoveredObject != null) {
sendAccessibilityEvent(mHoveredObject.id, AccessibilityEvent.TYPE_VIEW_HOVER_EXIT);
}
mHoveredNode = newNode;
mHoveredObject = newObject;
}
}
@Override
public void updateSemanticsTree(SemanticsNode[] nodes) {
Set<SemanticObject> updatedObjects = new HashSet<SemanticObject>();
Set<SemanticObject> removedObjects = new HashSet<SemanticObject>();
for (SemanticsNode node : nodes) {
updateSemanticsNode(node);
updateSemanticObject(node, updatedObjects, removedObjects);
sendAccessibilityEvent(node.id, AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED);
}
for (SemanticObject object : removedObjects) {
if (!updatedObjects.contains(object)) {
removeSemanticObject(object, updatedObjects);
}
}
}
private SemanticObject updateSemanticObject(SemanticsNode node,
Set<SemanticObject> updatedObjects,
Set<SemanticObject> removedObjects) {
SemanticObject object = mObjects.get(node.id);
if (object == null) {
object = new SemanticObject();
mObjects.put(node.id, object);
}
object.updateWith(node);
updatedObjects.add(object);
if (node.children != null) {
if (node.children.length == 0) {
if (object.children != null) {
removedObjects.addAll(object.children);
}
object.children = null;
} else {
if (object.children == null) {
object.children = new ArrayList<SemanticObject>(node.children.length);
} else {
removedObjects.addAll(object.children);
object.children.clear();
}
for (SemanticsNode childNode : node.children) {
SemanticObject childObject = updateSemanticObject(childNode, updatedObjects, removedObjects);
childObject.parent = object;
object.children.add(childObject);
}
}
}
if (node.geometry != null) {
// has to be done after children are updated
// since they also get marked dirty
object.invalidateGlobalGeometry();
}
return object;
}
private void sendAccessibilityEvent(int virtualViewId, int eventType) {
@ -236,52 +288,43 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
}
}
private PersistentAccessibilityNode updateSemanticsNode(SemanticsNode node) {
PersistentAccessibilityNode persistentNode = mTreeNodes.get(node.id);
if (persistentNode != null) {
persistentNode.update(node);
} else {
persistentNode = new PersistentAccessibilityNode(node);
mTreeNodes.put(node.id, persistentNode);
private void removeSemanticObject(SemanticObject object, Set<SemanticObject> updatedObjects) {
assert mObjects.containsKey(object.id);
assert mObjects.get(object.id) == object;
object.parent = null;
mObjects.remove(object.id);
if (mFocusedObject == object) {
sendAccessibilityEvent(mFocusedObject.id, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED);
mFocusedObject = null;
}
assert persistentNode != null;
return persistentNode;
}
void removePersistentNode(PersistentAccessibilityNode node) {
assert mTreeNodes.containsKey(node.id);
assert mTreeNodes.get(node.id).parent == null;
mTreeNodes.remove(node.id);
if (mFocusedNode == node) {
sendAccessibilityEvent(mFocusedNode.id, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED);
mFocusedNode = null;
if (mHoveredObject == object) {
mHoveredObject = null;
}
if (mHoveredNode == node) {
mHoveredNode = null;
}
if (node.children != null) {
for (PersistentAccessibilityNode child : node.children) {
removePersistentNode(child);
if (object.children != null) {
for (SemanticObject child : object.children) {
if (!updatedObjects.contains(child)) {
assert child.parent == object;
removeSemanticObject(child, updatedObjects);
}
}
}
}
void reset(SemanticsServer.Proxy newSemanticsServer) {
mTreeNodes.clear();
sendAccessibilityEvent(mFocusedNode.id, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED);
mFocusedNode = null;
mHoveredNode = null;
mObjects.clear();
sendAccessibilityEvent(mFocusedObject.id, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED);
mFocusedObject = null;
mHoveredObject = null;
mSemanticsServer.close();
sendAccessibilityEvent(0, AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED);
mSemanticsServer = newSemanticsServer;
mSemanticsServer.addSemanticsListener(this);
}
private class PersistentAccessibilityNode {
PersistentAccessibilityNode(SemanticsNode node) {
update(node);
}
void update(SemanticsNode node) {
private class SemanticObject {
SemanticObject() { }
void updateWith(SemanticsNode node) {
if (id == -1) {
id = node.id;
assert node.flags != null;
@ -291,13 +334,8 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
}
assert id == node.id;
if (node.flags != null) {
canBeTapped = node.flags.canBeTapped;
canBeLongPressed = node.flags.canBeLongPressed;
canBeScrolledHorizontally = node.flags.canBeScrolledHorizontally;
canBeScrolledVertically = node.flags.canBeScrolledVertically;
hasCheckedState = node.flags.hasCheckedState;
isChecked = node.flags.isChecked;
isAdjustable = node.flags.isAdjustable;
}
if (node.strings != null) {
label = node.strings.label;
@ -309,52 +347,53 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
width = node.geometry.width;
height = node.geometry.height;
}
if (node.children != null) {
List<PersistentAccessibilityNode> oldChildren = children;
if (oldChildren != null) {
for (PersistentAccessibilityNode child : oldChildren) {
assert child.parent != null;
child.parent = null;
if (node.actions != null) {
canBeTapped = false;
canBeLongPressed = false;
canBeScrolledHorizontally = false;
canBeScrolledVertically = false;
for (int action : node.actions) {
switch (action) {
case SemanticAction.TAP:
canBeTapped = true;
break;
case SemanticAction.LONG_PRESS:
canBeLongPressed = true;
break;
case SemanticAction.SCROLL_LEFT:
canBeScrolledHorizontally = true;
break;
case SemanticAction.SCROLL_RIGHT:
canBeScrolledHorizontally = true;
break;
case SemanticAction.SCROLL_UP:
canBeScrolledVertically = true;
break;
case SemanticAction.SCROLL_DOWN:
canBeScrolledVertically = true;
break;
case SemanticAction.INCREASE:
// Not implemented.
break;
case SemanticAction.DECREASE:
// Not implemented.
break;
}
}
if (node.children.length > 0) {
children = new ArrayList<PersistentAccessibilityNode>(node.children.length);
for (SemanticsNode childNode : node.children) {
PersistentAccessibilityNode child = AccessibilityBridge.this.updateSemanticsNode(childNode);
assert child != null;
child.parent = this;
children.add(child);
}
} else {
children = null;
}
if (oldChildren != null) {
for (PersistentAccessibilityNode child : oldChildren) {
if (child.parent == null) {
AccessibilityBridge.this.removePersistentNode(child);
}
}
}
}
if (node.geometry != null) {
// has to be done after children are updated
// since they also get marked dirty
invalidateGlobalGeometry();
}
}
// fields that we pass straight to the Android accessibility API
int id = -1;
PersistentAccessibilityNode parent;
SemanticObject parent;
boolean canBeTapped;
boolean canBeLongPressed;
boolean canBeScrolledHorizontally;
boolean canBeScrolledVertically;
boolean hasCheckedState;
boolean isChecked;
boolean isAdjustable;
String label;
List<PersistentAccessibilityNode> children;
List<SemanticObject> children;
// geometry, which we have to convert to global coordinates to send to Android
private float[] transform; // can be null, meaning identity transform
@ -369,10 +408,10 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
return;
}
geometryDirty = true;
// TODO(ianh): if we are the AccessibilityBridge.this.mFocusedNode
// TODO(ianh): if we are the AccessibilityBridge.this.mFocusedObject
// then we may have to unfocus and refocus ourselves to get Android to update the focus rect
if (children != null) {
for (PersistentAccessibilityNode child : children) {
for (SemanticObject child : children) {
child.invalidateGlobalGeometry();
}
}
@ -437,14 +476,14 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
return globalRect;
}
PersistentAccessibilityNode hitTest(int x, int y) {
SemanticObject hitTest(int x, int y) {
Rect rect = getGlobalRect();
if (!rect.contains(x, y))
return null;
if (children != null) {
for (int index = children.size()-1; index >= 0; index -= 1) {
PersistentAccessibilityNode child = children.get(index);
PersistentAccessibilityNode result = child.hitTest(x, y);
SemanticObject child = children.get(index);
SemanticObject result = child.hitTest(x, y);
if (result != null) {
return result;
}

12
sky/shell/platform/ios/framework/Source/FlutterView.mm
View File

@ -12,12 +12,11 @@
@end
@implementation FlutterView {
base::WeakPtr<sky::shell::AccessibilityBridge> _accessibilityBridge;
std::unique_ptr<sky::shell::AccessibilityBridge> _accessibilityBridge;
}
- (void)withAccessibility:(mojo::ServiceProvider*)serviceProvider {
auto bridge = new sky::shell::AccessibilityBridge(self, serviceProvider);
_accessibilityBridge = bridge->AsWeakPtr();
_accessibilityBridge.reset(new sky::shell::AccessibilityBridge(self, serviceProvider));
}
- (void)layoutSubviews {
@ -40,11 +39,4 @@
return YES;
}
- (void)dealloc {
delete _accessibilityBridge.get();
_accessibilityBridge.reset();
[super dealloc];
}
@end

41
sky/shell/platform/ios/framework/Source/accessibility_bridge.h
View File

@ -5,14 +5,11 @@
#ifndef SKY_SHELL_PLATFORM_IOS_FRAMEWORK_SOURCE_ACCESSIBILITY_BRIDGE_H_
#define SKY_SHELL_PLATFORM_IOS_FRAMEWORK_SOURCE_ACCESSIBILITY_BRIDGE_H_
#include <map>
#include <memory>
#include <string>
#include <vector>
#include <set>
#include <unordered_map>
#include "base/macros.h"
#include "base/memory/ref_counted.h"
#include "base/memory/weak_ptr.h"
#include "mojo/public/cpp/bindings/array.h"
#include "mojo/public/cpp/bindings/strong_binding.h"
#include "mojo/public/interfaces/application/service_provider.mojom.h"
@ -28,7 +25,7 @@ class AccessibilityBridge;
}
}
@interface AccessibilityNode : NSObject
@interface SemanticObject : NSObject
/**
* The globally unique identifier for this node.
@ -39,12 +36,7 @@ class AccessibilityBridge;
* The parent of this node in the node tree. Will be nil for the root node and
* during transient state changes.
*/
@property(nonatomic, readonly) AccessibilityNode* parent;
/**
* This node's children in the node tree.
*/
@property(nonatomic, readonly) NSArray<AccessibilityNode*>* children;
@property(nonatomic, assign) SemanticObject* parent;
- (instancetype)init __attribute__((unavailable("Use initWithBridge instead")));
- (instancetype)initWithBridge:(sky::shell::AccessibilityBridge*)bridge
@ -55,36 +47,29 @@ class AccessibilityBridge;
namespace sky {
namespace shell {
// Class that mediates communication between FlutterView and the Dart layer in
// order to provide accessibility features.
//
// The bridge is owned by the FlutterView that created it. It maintains a raw
// pointer back to the view to enable bidirectional communication with the view
// without introducing a circular reference. Since the strong binding herein may
// destroy the bridge, the view maintains its ownership via a weak reference.
class AccessibilityBridge final : public semantics::SemanticsListener {
public:
AccessibilityBridge(FlutterView*, mojo::ServiceProvider*);
~AccessibilityBridge() override;
void UpdateSemanticsTree(mojo::Array<semantics::SemanticsNodePtr>) override;
AccessibilityNode* UpdateNode(const semantics::SemanticsNodePtr& node);
void RemoveNode(AccessibilityNode* node);
base::WeakPtr<AccessibilityBridge> AsWeakPtr();
FlutterView* view() { return view_; }
semantics::SemanticsServer* server() { return semantics_server_.get(); }
private:
// See class docs above about ownership relationship
SemanticObject* UpdateSemanticObject(
const semantics::SemanticsNodePtr& node,
std::set<SemanticObject*>* updated_objects,
std::set<SemanticObject*>* removed_objects);
void RemoveSemanticObject(SemanticObject* node,
std::set<SemanticObject*>* updated_objects);
FlutterView* view_;
semantics::SemanticsServerPtr semantics_server_;
NSMutableDictionary<NSNumber*, AccessibilityNode*>* nodes_;
std::unordered_map<int, SemanticObject*> objects_;
mojo::StrongBinding<semantics::SemanticsListener> binding_;
base::WeakPtrFactory<AccessibilityBridge> weak_factory_;
mojo::Binding<semantics::SemanticsListener> binding_;
DISALLOW_COPY_AND_ASSIGN(AccessibilityBridge);
};

273
sky/shell/platform/ios/framework/Source/accessibility_bridge.mm
View File

@ -5,12 +5,14 @@
#include "sky/shell/platform/ios/framework/Source/accessibility_bridge.h"
#import <UIKit/UIKit.h>
#include <vector>
#include "base/logging.h"
#include "mojo/public/cpp/application/connect.h"
namespace {
static const uint32_t RootNodeId = 0;
constexpr uint32_t kRootNodeId = 0;
// Contains better abstractions than the raw Mojo data structure
struct Geometry {
@ -22,19 +24,25 @@ struct Geometry {
return *this;
}
SkMatrix44 transform =
SkMatrix44(SkMatrix44::Identity_Constructor::kIdentity_Constructor);
SkMatrix44 transform = SkMatrix44(SkMatrix44::kIdentity_Constructor);
SkRect rect;
};
} // anonymous namespace
} // namespace
@implementation AccessibilityNode {
@implementation SemanticObject {
sky::shell::AccessibilityBridge* _bridge;
semantics::SemanticFlagsPtr _flags;
semantics::SemanticStringsPtr _strings;
Geometry _geometry;
bool _canBeTapped;
bool _canBeLongPressed;
bool _canBeScrolledHorizontally;
bool _canBeScrolledVertically;
bool _canBeAdjusted;
std::vector<SemanticObject*> _children;
}
#pragma mark - Override base class designated initializers
@ -51,7 +59,7 @@ struct Geometry {
- (instancetype)initWithBridge:(sky::shell::AccessibilityBridge*)bridge
uid:(uint32_t)uid {
DCHECK(bridge != nil) << "bridge must be set";
DCHECK(uid >= RootNodeId);
DCHECK(uid >= kRootNodeId);
self = [super init];
if (self) {
@ -62,53 +70,61 @@ struct Geometry {
return self;
}
#pragma mark - Semantics node methods
#pragma mark - Semantic object methods
- (void)update:(const semantics::SemanticsNodePtr&)mojoNode {
DCHECK(_uid == mojoNode->id);
- (void)updateWith:(const semantics::SemanticsNodePtr&)node {
DCHECK(_uid == node->id);
if (!mojoNode->flags.is_null()) {
_flags = mojoNode->flags.Pass();
if (!node->flags.is_null()) {
_flags = node->flags.Pass();
}
if (!mojoNode->strings.is_null()) {
_strings = mojoNode->strings.Pass();
if (!node->strings.is_null()) {
_strings = node->strings.Pass();
}
if (!mojoNode->geometry.is_null()) {
_geometry = mojoNode->geometry;
if (!node->geometry.is_null()) {
_geometry = node->geometry;
}
if (!mojoNode->children.is_null()) {
// Mark existing children as orphans
NSArray* oldChildren = _children;
for (AccessibilityNode* child in oldChildren) {
DCHECK(child->_parent != nil);
child->_parent = nil;
}
// Set the new list of children
NSMutableArray* children = [[NSMutableArray alloc] init];
_children = children;
for (const semantics::SemanticsNodePtr& mojoChild : mojoNode->children) {
AccessibilityNode* child = _bridge->UpdateNode(mojoChild);
child->_parent = self;
[children insertObject:child atIndex:0];
}
// Remove those children that are still marked as orphans
for (AccessibilityNode* child in oldChildren) {
if (child->_parent == nil) {
_bridge->RemoveNode(child);
if (!node->actions.is_null()) {
_canBeTapped = false;
_canBeLongPressed = false;
_canBeScrolledHorizontally = false;
_canBeScrolledVertically = false;
for (int action : node->actions) {
switch (static_cast<semantics::SemanticAction>(action)) {
case semantics::SemanticAction::TAP:
_canBeTapped = true;
break;
case semantics::SemanticAction::LONG_PRESS:
_canBeLongPressed = true;
break;
case semantics::SemanticAction::SCROLL_LEFT:
case semantics::SemanticAction::SCROLL_RIGHT:
_canBeScrolledHorizontally = true;
break;
case semantics::SemanticAction::SCROLL_UP:
case semantics::SemanticAction::SCROLL_DOWN:
_canBeScrolledVertically = true;
break;
case semantics::SemanticAction::INCREASE:
case semantics::SemanticAction::DECREASE:
_canBeAdjusted = true;
break;
}
}
[oldChildren release];
}
}
- (void)remove {
_parent = nil;
_bridge->RemoveNode(self);
- (std::vector<SemanticObject*>*)children {
return &_children;
}
- (void)neuter {
_bridge = nullptr;
_children.clear();
self.parent = nil;
}
#pragma mark - UIAccessibility overrides
@ -117,27 +133,30 @@ struct Geometry {
// Note: hit detection will only apply to elements that report
// -isAccessibilityElement of YES. The framework will continue scanning the
// entire element tree looking for such a hit.
return (_flags->canBeTapped || _children.count == 0);
return _canBeTapped || _children.empty();
}
- (NSString*)accessibilityLabel {
return (_strings.is_null() || _strings->label.get().empty())
? nil
: @(_strings->label.data());
if (_strings.is_null() || _strings->label.get().empty()) {
return nil;
}
return @(_strings->label.data());
}
- (UIAccessibilityTraits)accessibilityTraits {
UIAccessibilityTraits traits = UIAccessibilityTraitNone;
if (_flags->canBeTapped)
if (_canBeTapped) {
traits |= UIAccessibilityTraitButton;
if (_flags->isAdjustable)
}
if (_canBeAdjusted) {
traits |= UIAccessibilityTraitAdjustable;
}
return traits;
}
- (CGRect)accessibilityFrame {
SkMatrix44 globalTransform = _geometry.transform;
for (AccessibilityNode* parent = _parent; parent; parent = parent.parent) {
for (SemanticObject* parent = _parent; parent; parent = parent.parent) {
globalTransform = globalTransform * parent->_geometry.transform;
}
@ -159,22 +178,28 @@ struct Geometry {
#pragma mark - UIAccessibilityElement protocol
- (id)accessibilityContainer {
return (_uid == RootNodeId) ? _bridge->view() : _parent;
return (_uid == kRootNodeId) ? _bridge->view() : _parent;
}
#pragma mark - UIAccessibilityContainer overrides
- (NSInteger)accessibilityElementCount {
return _children.count;
return (NSInteger)_children.size();
}
- (nullable id)accessibilityElementAtIndex:(NSInteger)index {
return (index < 0 || index >= (NSInteger)_children.count ? nil
: _children[index]);
if (index < 0 || index >= (NSInteger)_children.size()) {
return nil;
}
return _children[index];
}
- (NSInteger)indexOfAccessibilityElement:(id)element {
return (_children == nil) ? NSNotFound : [_children indexOfObject:element];
auto it = std::find(_children.begin(), _children.end(), element);
if (it == _children.end()) {
return NSNotFound;
}
return it - _children.begin();
}
#pragma mark - UIAccessibilityAction overrides
@ -185,11 +210,15 @@ struct Geometry {
}
- (void)accessibilityIncrement {
// TODO(tvolkert): Implement
if (_canBeAdjusted) {
_bridge->server()->PerformAction(_uid, semantics::SemanticAction::INCREASE);
}
}
- (void)accessibilityDecrement {
// TODO(tvolkert): Implement
if (_canBeAdjusted) {
_bridge->server()->PerformAction(_uid, semantics::SemanticAction::DECREASE);
}
}
- (BOOL)accessibilityScroll:(UIAccessibilityScrollDirection)direction {
@ -197,17 +226,18 @@ struct Geometry {
switch (direction) {
case UIAccessibilityScrollDirectionRight:
case UIAccessibilityScrollDirectionLeft:
canBeScrolled = _flags->canBeScrolledHorizontally;
canBeScrolled = _canBeScrolledHorizontally;
break;
case UIAccessibilityScrollDirectionUp:
case UIAccessibilityScrollDirectionDown:
canBeScrolled = _flags->canBeScrolledVertically;
canBeScrolled = _canBeScrolledVertically;
break;
default:
// Note: page turning of reading content is not currently supported
// (UIAccessibilityScrollDirectionNext,
// UIAccessibilityScrollDirectionPrevious)
canBeScrolled = NO;
break;
}
if (!canBeScrolled) {
@ -216,16 +246,16 @@ struct Geometry {
switch (direction) {
case UIAccessibilityScrollDirectionRight:
_bridge->server()->ScrollRight(_uid);
_bridge->server()->PerformAction(_uid, semantics::SemanticAction::SCROLL_RIGHT);
break;
case UIAccessibilityScrollDirectionLeft:
_bridge->server()->ScrollLeft(_uid);
_bridge->server()->PerformAction(_uid, semantics::SemanticAction::SCROLL_LEFT);
break;
case UIAccessibilityScrollDirectionUp:
_bridge->server()->ScrollDown(_uid);
_bridge->server()->PerformAction(_uid, semantics::SemanticAction::SCROLL_UP);
break;
case UIAccessibilityScrollDirectionDown:
_bridge->server()->ScrollUp(_uid);
_bridge->server()->PerformAction(_uid, semantics::SemanticAction::SCROLL_DOWN);
break;
default:
DCHECK(false) << "Unsupported scroll direction: " << direction;
@ -246,13 +276,6 @@ struct Geometry {
return NO;
}
#pragma mark - Misc
- (void)dealloc {
[_children release];
[super dealloc];
}
@end
#pragma mark - AccessibilityBridge impl
@ -262,59 +285,83 @@ namespace shell {
AccessibilityBridge::AccessibilityBridge(FlutterView* view,
mojo::ServiceProvider* serviceProvider)
: view_(view), binding_(this), weak_factory_(this) {
: view_(view), binding_(this) {
mojo::ConnectToService(serviceProvider, mojo::GetProxy(&semantics_server_));
mojo::InterfaceHandle<semantics::SemanticsListener> listener;
binding_.Bind(&listener);
semantics_server_->AddSemanticsListener(listener.Pass());
nodes_ = [[NSMutableDictionary alloc] init];
}
void AccessibilityBridge::UpdateSemanticsTree(
mojo::Array<semantics::SemanticsNodePtr> mojoNodes) {
for (const semantics::SemanticsNodePtr& mojoNode : mojoNodes) {
auto node = UpdateNode(mojoNode);
if (mojoNode->id == RootNodeId && view_.accessibilityElements == nil) {
view_.accessibilityElements = @[ node ];
}
}
DCHECK(view_.accessibilityElements != nil);
UIAccessibilityPostNotification(UIAccessibilityLayoutChangedNotification,
nil);
}
base::WeakPtr<AccessibilityBridge> AccessibilityBridge::AsWeakPtr() {
return weak_factory_.GetWeakPtr();
}
AccessibilityNode* AccessibilityBridge::UpdateNode(
const semantics::SemanticsNodePtr& mojoNode) {
AccessibilityNode* node = nodes_[@(mojoNode->id)];
if (node == nil) {
node = [[AccessibilityNode alloc] initWithBridge:this uid:mojoNode->id];
DCHECK(nodes_ != nil);
nodes_[@(mojoNode->id)] = node;
[node release];
}
[node update:mojoNode];
return node;
}
void AccessibilityBridge::RemoveNode(AccessibilityNode* node) {
[node retain];
DCHECK(nodes_[@(node.uid)] != nil);
DCHECK(nodes_[@(node.uid)].parent == nil);
[nodes_ removeObjectForKey:@(node.uid)];
for (AccessibilityNode* child in node.children) {
[child remove];
}
[node release];
}
AccessibilityBridge::~AccessibilityBridge() {
[nodes_ release];
for (const auto& entry : objects_) {
SemanticObject* object = entry.second;
[object neuter];
[object release];
}
}
void AccessibilityBridge::UpdateSemanticsTree(
mojo::Array<semantics::SemanticsNodePtr> nodes) {
std::set<SemanticObject*> updated_objects;
std::set<SemanticObject*> removed_objects;
for (const semantics::SemanticsNodePtr& node : nodes) {
UpdateSemanticObject(node, &updated_objects, &removed_objects);
}
for (SemanticObject* object : removed_objects) {
if (!updated_objects.count(object)) {
RemoveSemanticObject(object, &updated_objects);
}
}
if (!view_.accessibilityElements) {
view_.accessibilityElements = @[ objects_[kRootNodeId] ];
}
UIAccessibilityPostNotification(
UIAccessibilityLayoutChangedNotification, nil);
}
SemanticObject* AccessibilityBridge::UpdateSemanticObject(
const semantics::SemanticsNodePtr& node,
std::set<SemanticObject*>* updated_objects,
std::set<SemanticObject*>* removed_objects) {
SemanticObject* object = objects_[node->id];
if (!object) {
object = [[SemanticObject alloc] initWithBridge:this uid:node->id];
objects_[node->id] = object;
}
[object updateWith:node];
updated_objects->insert(object);
if (!node->children.is_null()) {
std::vector<SemanticObject*>* children = [object children];
removed_objects->insert(children->begin(), children->end());
children->clear();
children->reserve(node->children.size());
for (const auto& child_node : node->children) {
SemanticObject* child_object =
UpdateSemanticObject(child_node, updated_objects, removed_objects);
child_object.parent = object;
children->push_back(child_object);
}
}
return object;
}
void AccessibilityBridge::RemoveSemanticObject(
SemanticObject* object,
std::set<SemanticObject*>* updated_objects) {
DCHECK(objects_[object.uid] == object);
objects_.erase(object.uid);
for (SemanticObject* child : *[object children]) {
if (!updated_objects->count(child)) {
DCHECK(child.parent == object);
child.parent = nil;
RemoveSemanticObject(child, updated_objects);
}
}
[object neuter];
[object release];
}
} // namespace shell