From 71ce354a447dc3384c4e8b0f031eaa9f313d29e9 Mon Sep 17 00:00:00 2001 From: Adam Barth Date: Thu, 30 Jun 2016 16:49:04 -0700 Subject: [PATCH] Harmonize Android and iOS accessibility bridges (#2777) These classes now use the same terminology and work in the same way. Also, change semantics.mojom to use an enumeration of actions instead of having a separate method per action. This will hopefully scale better. --- sky/services/semantics/semantics.mojom | 34 +- .../io/flutter/view/AccessibilityBridge.java | 295 ++++++++++-------- .../ios/framework/Source/FlutterView.mm | 12 +- .../framework/Source/accessibility_bridge.h | 41 +-- .../framework/Source/accessibility_bridge.mm | 273 +++++++++------- 5 files changed, 360 insertions(+), 295 deletions(-) diff --git a/sky/services/semantics/semantics.mojom b/sky/services/semantics/semantics.mojom index a7e099d1202..d4519864e7e 100644 --- a/sky/services/semantics/semantics.mojom +++ b/sky/services/semantics/semantics.mojom @@ -5,6 +5,17 @@ [DartPackage="sky_services"] module semantics; +enum SemanticAction { + TAP, + LONG_PRESS, + SCROLL_LEFT, + SCROLL_RIGHT, + SCROLL_UP, + SCROLL_DOWN, + INCREASE, + DECREASE, +}; + struct SemanticsNode { uint32 id; @@ -12,19 +23,17 @@ struct SemanticsNode { SemanticFlags? flags; SemanticStrings? strings; SemanticGeometry? geometry; + // TODO(abarth): Switch to array once that works. + // See https://github.com/domokit/mojo/issues/799 + array? actions; array? children; }; struct SemanticFlags { // This is intended to just be booleans, so that it can be extended // over time yet still be packed tightly. - bool canBeTapped = false; - bool canBeLongPressed = false; - bool canBeScrolledHorizontally = false; - bool canBeScrolledVertically = false; - bool hasCheckedState = false; // whether isChecked is relevant - bool isChecked = false; - bool isAdjustable = false; + bool has_checked_state = false; // whether is_checked is relevant + bool is_checked = false; }; struct SemanticStrings { @@ -56,7 +65,7 @@ struct SemanticGeometry { }; interface SemanticsListener { - // The OS side, invoked from the app. + // The engine side, invoked from the app. UpdateSemanticsTree(array nodes); }; @@ -64,12 +73,5 @@ interface SemanticsListener { interface SemanticsServer { // The app side, invoked from the engine. AddSemanticsListener(SemanticsListener listener); - Tap(uint32 nodeID); - LongPress(uint32 nodeID); - ScrollLeft(uint32 nodeID); - ScrollRight(uint32 nodeID); - ScrollUp(uint32 nodeID); - ScrollDown(uint32 nodeID); - AdjustIncrease(uint32 nodeID); - AdjustDecrease(uint32 nodeID); + PerformAction(uint32 nodeID, SemanticAction action); }; diff --git a/sky/shell/platform/android/io/flutter/view/AccessibilityBridge.java b/sky/shell/platform/android/io/flutter/view/AccessibilityBridge.java index b35f11d975a..73b08ccb5f9 100644 --- a/sky/shell/platform/android/io/flutter/view/AccessibilityBridge.java +++ b/sky/shell/platform/android/io/flutter/view/AccessibilityBridge.java @@ -14,6 +14,7 @@ import android.view.accessibility.AccessibilityNodeInfo; import android.view.accessibility.AccessibilityNodeProvider; import org.chromium.mojo.system.MojoException; +import org.chromium.mojom.semantics.SemanticAction; import org.chromium.mojom.semantics.SemanticsListener; import org.chromium.mojom.semantics.SemanticsNode; import org.chromium.mojom.semantics.SemanticsServer; @@ -21,22 +22,24 @@ import org.chromium.mojom.sky.ViewportMetrics; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; class AccessibilityBridge extends AccessibilityNodeProvider implements SemanticsListener { - private Map mTreeNodes; + private Map mObjects; private FlutterView mOwner; private SemanticsServer.Proxy mSemanticsServer; private boolean mAccessibilityEnabled = false; - private PersistentAccessibilityNode mFocusedNode; - private PersistentAccessibilityNode mHoveredNode; + private SemanticObject mFocusedObject; + private SemanticObject mHoveredObject; AccessibilityBridge(FlutterView owner, SemanticsServer.Proxy semanticsServer) { assert owner != null; assert semanticsServer != null; mOwner = owner; - mTreeNodes = new HashMap(); + mObjects = new HashMap(); mSemanticsServer = semanticsServer; mSemanticsServer.addSemanticsListener(this); } @@ -47,17 +50,16 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics @Override public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId) { - if (virtualViewId == View.NO_ID) { AccessibilityNodeInfo result = AccessibilityNodeInfo.obtain(mOwner); mOwner.onInitializeAccessibilityNodeInfo(result); - if (mTreeNodes.containsKey(0)) + if (mObjects.containsKey(0)) result.addChild(mOwner, 0); return result; } - PersistentAccessibilityNode node = mTreeNodes.get(virtualViewId); - if (node == null) + SemanticObject object = mObjects.get(virtualViewId); + if (object == null) return null; AccessibilityNodeInfo result = AccessibilityNodeInfo.obtain(mOwner, virtualViewId); @@ -65,17 +67,17 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics result.setClassName("Flutter"); // Prettier than the more conventional node.getClass().getName() result.setSource(mOwner, virtualViewId); - if (node.parent != null) { - assert node.id > 0; - result.setParent(mOwner, node.parent.id); + if (object.parent != null) { + assert object.id > 0; + result.setParent(mOwner, object.parent.id); } else { - assert node.id == 0; + assert object.id == 0; result.setParent(mOwner); } - Rect bounds = node.getGlobalRect(); - if (node.parent != null) { - Rect parentBounds = node.parent.getGlobalRect(); + Rect bounds = object.getGlobalRect(); + if (object.parent != null) { + Rect parentBounds = object.parent.getGlobalRect(); Rect boundsInParent = new Rect(bounds); boundsInParent.offset(-parentBounds.left, -parentBounds.top); result.setBoundsInParent(boundsInParent); @@ -86,15 +88,15 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics result.setVisibleToUser(true); result.setEnabled(true); // TODO(ianh): Expose disabled subtrees - if (node.canBeTapped) { + if (object.canBeTapped) { result.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_CLICK); result.setClickable(true); } - if (node.canBeLongPressed) { + if (object.canBeLongPressed) { result.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_LONG_CLICK); result.setLongClickable(true); } - if (node.canBeScrolledHorizontally || node.canBeScrolledVertically) { + if (object.canBeScrolledHorizontally || object.canBeScrolledVertically) { // TODO(ianh): Once we're on SDK v23+, call addAction to // expose AccessibilityAction.ACTION_SCROLL_LEFT, _RIGHT, // _UP, and _DOWN when appropriate. @@ -104,9 +106,9 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics result.setScrollable(true); } - result.setCheckable(node.hasCheckedState); - result.setChecked(node.isChecked); - result.setText(node.label); + result.setCheckable(object.hasCheckedState); + result.setChecked(object.isChecked); + result.setText(object.label); // TODO(ianh): use setTraversalBefore/setTraversalAfter to set // the relative order of the views. For each set of siblings, @@ -115,14 +117,14 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics // width, and finally by list order. // Accessibility Focus - if (mFocusedNode != null && mFocusedNode.id == virtualViewId) { + if (mFocusedObject != null && mFocusedObject.id == virtualViewId) { result.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_CLEAR_ACCESSIBILITY_FOCUS); } else { result.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_ACCESSIBILITY_FOCUS); } - if (node.children != null) { - for (PersistentAccessibilityNode child : node.children) { + if (object.children != null) { + for (SemanticObject child : object.children) { result.addChild(mOwner, child.id); } } @@ -132,35 +134,36 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics @Override public boolean performAction(int virtualViewId, int action, Bundle arguments) { - PersistentAccessibilityNode node = mTreeNodes.get(virtualViewId); - if (node == null) + SemanticObject object = mObjects.get(virtualViewId); + if (object == null) { return false; + } switch (action) { case AccessibilityNodeInfo.ACTION_CLICK: { - mSemanticsServer.tap(virtualViewId); + mSemanticsServer.performAction(virtualViewId, SemanticAction.TAP); return true; } case AccessibilityNodeInfo.ACTION_LONG_CLICK: { - mSemanticsServer.longPress(virtualViewId); + mSemanticsServer.performAction(virtualViewId, SemanticAction.LONG_PRESS); return true; } case AccessibilityNodeInfo.ACTION_SCROLL_BACKWARD: { - if (node.canBeScrolledVertically) { - mSemanticsServer.scrollUp(virtualViewId); - } else if (node.canBeScrolledHorizontally) { + if (object.canBeScrolledVertically) { + mSemanticsServer.performAction(virtualViewId, SemanticAction.SCROLL_UP); + } else if (object.canBeScrolledHorizontally) { // TODO(ianh): bidi support - mSemanticsServer.scrollLeft(virtualViewId); + mSemanticsServer.performAction(virtualViewId, SemanticAction.SCROLL_LEFT); } else { return false; } return true; } case AccessibilityNodeInfo.ACTION_SCROLL_FORWARD: { - if (node.canBeScrolledVertically) { - mSemanticsServer.scrollDown(virtualViewId); - } else if (node.canBeScrolledHorizontally) { + if (object.canBeScrolledVertically) { + mSemanticsServer.performAction(virtualViewId, SemanticAction.SCROLL_DOWN); + } else if (object.canBeScrolledHorizontally) { // TODO(ianh): bidi support - mSemanticsServer.scrollRight(virtualViewId); + mSemanticsServer.performAction(virtualViewId, SemanticAction.SCROLL_RIGHT); } else { return false; } @@ -168,58 +171,107 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics } case AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS: { sendAccessibilityEvent(virtualViewId, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED); - mFocusedNode = null; + mFocusedObject = null; return true; } case AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS: { sendAccessibilityEvent(virtualViewId, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED); - if (mFocusedNode == null) { + if (mFocusedObject == null) { // When Android focuses a node, it doesn't invalidate the view. // (It does when it sends ACTION_CLEAR_ACCESSIBILITY_FOCUS, so // we only have to worry about this when the focused node is null.) mOwner.invalidate(); } - mFocusedNode = node; + mFocusedObject = object; return true; } } - // TODO(ianh): Implement left/right/up/down scrolling return false; } // TODO(ianh): implement findAccessibilityNodeInfosByText() // TODO(ianh): implement findFocus() + private SemanticObject getRootObject() { + return mObjects.get(0); + } + void handleTouchExplorationExit() { - if (mHoveredNode != null) { - sendAccessibilityEvent(mHoveredNode.id, AccessibilityEvent.TYPE_VIEW_HOVER_EXIT); - mHoveredNode = null; + if (mHoveredObject != null) { + sendAccessibilityEvent(mHoveredObject.id, AccessibilityEvent.TYPE_VIEW_HOVER_EXIT); + mHoveredObject = null; } } void handleTouchExploration(float x, float y) { - if (mTreeNodes.isEmpty()) + if (mObjects.isEmpty()) { return; - assert mTreeNodes.containsKey(0); - PersistentAccessibilityNode newNode = mTreeNodes.get(0).hitTest(Math.round(x), Math.round(y)); - if (newNode != mHoveredNode) { + } + assert mObjects.containsKey(0); + SemanticObject newObject = getRootObject().hitTest(Math.round(x), Math.round(y)); + if (newObject != mHoveredObject) { // sending ENTER before EXIT is how Android wants it - if (newNode != null) { - sendAccessibilityEvent(newNode.id, AccessibilityEvent.TYPE_VIEW_HOVER_ENTER); + if (newObject != null) { + sendAccessibilityEvent(newObject.id, AccessibilityEvent.TYPE_VIEW_HOVER_ENTER); } - if (mHoveredNode != null) { - sendAccessibilityEvent(mHoveredNode.id, AccessibilityEvent.TYPE_VIEW_HOVER_EXIT); + if (mHoveredObject != null) { + sendAccessibilityEvent(mHoveredObject.id, AccessibilityEvent.TYPE_VIEW_HOVER_EXIT); } - mHoveredNode = newNode; + mHoveredObject = newObject; } } @Override public void updateSemanticsTree(SemanticsNode[] nodes) { + Set updatedObjects = new HashSet(); + Set removedObjects = new HashSet(); for (SemanticsNode node : nodes) { - updateSemanticsNode(node); + updateSemanticObject(node, updatedObjects, removedObjects); sendAccessibilityEvent(node.id, AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED); } + for (SemanticObject object : removedObjects) { + if (!updatedObjects.contains(object)) { + removeSemanticObject(object, updatedObjects); + } + } + } + + private SemanticObject updateSemanticObject(SemanticsNode node, + Set updatedObjects, + Set removedObjects) { + SemanticObject object = mObjects.get(node.id); + if (object == null) { + object = new SemanticObject(); + mObjects.put(node.id, object); + } + object.updateWith(node); + updatedObjects.add(object); + if (node.children != null) { + if (node.children.length == 0) { + if (object.children != null) { + removedObjects.addAll(object.children); + } + object.children = null; + } else { + if (object.children == null) { + object.children = new ArrayList(node.children.length); + } else { + removedObjects.addAll(object.children); + object.children.clear(); + } + for (SemanticsNode childNode : node.children) { + SemanticObject childObject = updateSemanticObject(childNode, updatedObjects, removedObjects); + childObject.parent = object; + object.children.add(childObject); + } + } + } + if (node.geometry != null) { + // has to be done after children are updated + // since they also get marked dirty + object.invalidateGlobalGeometry(); + } + return object; } private void sendAccessibilityEvent(int virtualViewId, int eventType) { @@ -236,52 +288,43 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics } } - private PersistentAccessibilityNode updateSemanticsNode(SemanticsNode node) { - PersistentAccessibilityNode persistentNode = mTreeNodes.get(node.id); - if (persistentNode != null) { - persistentNode.update(node); - } else { - persistentNode = new PersistentAccessibilityNode(node); - mTreeNodes.put(node.id, persistentNode); + private void removeSemanticObject(SemanticObject object, Set updatedObjects) { + assert mObjects.containsKey(object.id); + assert mObjects.get(object.id) == object; + object.parent = null; + mObjects.remove(object.id); + if (mFocusedObject == object) { + sendAccessibilityEvent(mFocusedObject.id, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED); + mFocusedObject = null; } - assert persistentNode != null; - return persistentNode; - } - - void removePersistentNode(PersistentAccessibilityNode node) { - assert mTreeNodes.containsKey(node.id); - assert mTreeNodes.get(node.id).parent == null; - mTreeNodes.remove(node.id); - if (mFocusedNode == node) { - sendAccessibilityEvent(mFocusedNode.id, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED); - mFocusedNode = null; + if (mHoveredObject == object) { + mHoveredObject = null; } - if (mHoveredNode == node) { - mHoveredNode = null; - } - if (node.children != null) { - for (PersistentAccessibilityNode child : node.children) { - removePersistentNode(child); + if (object.children != null) { + for (SemanticObject child : object.children) { + if (!updatedObjects.contains(child)) { + assert child.parent == object; + removeSemanticObject(child, updatedObjects); + } } } } void reset(SemanticsServer.Proxy newSemanticsServer) { - mTreeNodes.clear(); - sendAccessibilityEvent(mFocusedNode.id, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED); - mFocusedNode = null; - mHoveredNode = null; + mObjects.clear(); + sendAccessibilityEvent(mFocusedObject.id, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED); + mFocusedObject = null; + mHoveredObject = null; mSemanticsServer.close(); sendAccessibilityEvent(0, AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED); mSemanticsServer = newSemanticsServer; mSemanticsServer.addSemanticsListener(this); } - private class PersistentAccessibilityNode { - PersistentAccessibilityNode(SemanticsNode node) { - update(node); - } - void update(SemanticsNode node) { + private class SemanticObject { + SemanticObject() { } + + void updateWith(SemanticsNode node) { if (id == -1) { id = node.id; assert node.flags != null; @@ -291,13 +334,8 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics } assert id == node.id; if (node.flags != null) { - canBeTapped = node.flags.canBeTapped; - canBeLongPressed = node.flags.canBeLongPressed; - canBeScrolledHorizontally = node.flags.canBeScrolledHorizontally; - canBeScrolledVertically = node.flags.canBeScrolledVertically; hasCheckedState = node.flags.hasCheckedState; isChecked = node.flags.isChecked; - isAdjustable = node.flags.isAdjustable; } if (node.strings != null) { label = node.strings.label; @@ -309,52 +347,53 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics width = node.geometry.width; height = node.geometry.height; } - if (node.children != null) { - List oldChildren = children; - if (oldChildren != null) { - for (PersistentAccessibilityNode child : oldChildren) { - assert child.parent != null; - child.parent = null; + if (node.actions != null) { + canBeTapped = false; + canBeLongPressed = false; + canBeScrolledHorizontally = false; + canBeScrolledVertically = false; + for (int action : node.actions) { + switch (action) { + case SemanticAction.TAP: + canBeTapped = true; + break; + case SemanticAction.LONG_PRESS: + canBeLongPressed = true; + break; + case SemanticAction.SCROLL_LEFT: + canBeScrolledHorizontally = true; + break; + case SemanticAction.SCROLL_RIGHT: + canBeScrolledHorizontally = true; + break; + case SemanticAction.SCROLL_UP: + canBeScrolledVertically = true; + break; + case SemanticAction.SCROLL_DOWN: + canBeScrolledVertically = true; + break; + case SemanticAction.INCREASE: + // Not implemented. + break; + case SemanticAction.DECREASE: + // Not implemented. + break; } } - if (node.children.length > 0) { - children = new ArrayList(node.children.length); - for (SemanticsNode childNode : node.children) { - PersistentAccessibilityNode child = AccessibilityBridge.this.updateSemanticsNode(childNode); - assert child != null; - child.parent = this; - children.add(child); - } - } else { - children = null; - } - if (oldChildren != null) { - for (PersistentAccessibilityNode child : oldChildren) { - if (child.parent == null) { - AccessibilityBridge.this.removePersistentNode(child); - } - } - } - } - if (node.geometry != null) { - // has to be done after children are updated - // since they also get marked dirty - invalidateGlobalGeometry(); } } // fields that we pass straight to the Android accessibility API int id = -1; - PersistentAccessibilityNode parent; + SemanticObject parent; boolean canBeTapped; boolean canBeLongPressed; boolean canBeScrolledHorizontally; boolean canBeScrolledVertically; boolean hasCheckedState; boolean isChecked; - boolean isAdjustable; String label; - List children; + List children; // geometry, which we have to convert to global coordinates to send to Android private float[] transform; // can be null, meaning identity transform @@ -369,10 +408,10 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics return; } geometryDirty = true; - // TODO(ianh): if we are the AccessibilityBridge.this.mFocusedNode + // TODO(ianh): if we are the AccessibilityBridge.this.mFocusedObject // then we may have to unfocus and refocus ourselves to get Android to update the focus rect if (children != null) { - for (PersistentAccessibilityNode child : children) { + for (SemanticObject child : children) { child.invalidateGlobalGeometry(); } } @@ -437,14 +476,14 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics return globalRect; } - PersistentAccessibilityNode hitTest(int x, int y) { + SemanticObject hitTest(int x, int y) { Rect rect = getGlobalRect(); if (!rect.contains(x, y)) return null; if (children != null) { for (int index = children.size()-1; index >= 0; index -= 1) { - PersistentAccessibilityNode child = children.get(index); - PersistentAccessibilityNode result = child.hitTest(x, y); + SemanticObject child = children.get(index); + SemanticObject result = child.hitTest(x, y); if (result != null) { return result; } diff --git a/sky/shell/platform/ios/framework/Source/FlutterView.mm b/sky/shell/platform/ios/framework/Source/FlutterView.mm index 536fd034784..97751012f39 100644 --- a/sky/shell/platform/ios/framework/Source/FlutterView.mm +++ b/sky/shell/platform/ios/framework/Source/FlutterView.mm @@ -12,12 +12,11 @@ @end @implementation FlutterView { - base::WeakPtr _accessibilityBridge; + std::unique_ptr _accessibilityBridge; } - (void)withAccessibility:(mojo::ServiceProvider*)serviceProvider { - auto bridge = new sky::shell::AccessibilityBridge(self, serviceProvider); - _accessibilityBridge = bridge->AsWeakPtr(); + _accessibilityBridge.reset(new sky::shell::AccessibilityBridge(self, serviceProvider)); } - (void)layoutSubviews { @@ -40,11 +39,4 @@ return YES; } -- (void)dealloc { - delete _accessibilityBridge.get(); - _accessibilityBridge.reset(); - - [super dealloc]; -} - @end diff --git a/sky/shell/platform/ios/framework/Source/accessibility_bridge.h b/sky/shell/platform/ios/framework/Source/accessibility_bridge.h index ee30e912ae2..b1f69708401 100644 --- a/sky/shell/platform/ios/framework/Source/accessibility_bridge.h +++ b/sky/shell/platform/ios/framework/Source/accessibility_bridge.h @@ -5,14 +5,11 @@ #ifndef SKY_SHELL_PLATFORM_IOS_FRAMEWORK_SOURCE_ACCESSIBILITY_BRIDGE_H_ #define SKY_SHELL_PLATFORM_IOS_FRAMEWORK_SOURCE_ACCESSIBILITY_BRIDGE_H_ -#include #include -#include -#include +#include +#include #include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "base/memory/weak_ptr.h" #include "mojo/public/cpp/bindings/array.h" #include "mojo/public/cpp/bindings/strong_binding.h" #include "mojo/public/interfaces/application/service_provider.mojom.h" @@ -28,7 +25,7 @@ class AccessibilityBridge; } } -@interface AccessibilityNode : NSObject +@interface SemanticObject : NSObject /** * The globally unique identifier for this node. @@ -39,12 +36,7 @@ class AccessibilityBridge; * The parent of this node in the node tree. Will be nil for the root node and * during transient state changes. */ -@property(nonatomic, readonly) AccessibilityNode* parent; - -/** - * This node's children in the node tree. - */ -@property(nonatomic, readonly) NSArray* children; +@property(nonatomic, assign) SemanticObject* parent; - (instancetype)init __attribute__((unavailable("Use initWithBridge instead"))); - (instancetype)initWithBridge:(sky::shell::AccessibilityBridge*)bridge @@ -55,36 +47,29 @@ class AccessibilityBridge; namespace sky { namespace shell { -// Class that mediates communication between FlutterView and the Dart layer in -// order to provide accessibility features. -// -// The bridge is owned by the FlutterView that created it. It maintains a raw -// pointer back to the view to enable bidirectional communication with the view -// without introducing a circular reference. Since the strong binding herein may -// destroy the bridge, the view maintains its ownership via a weak reference. class AccessibilityBridge final : public semantics::SemanticsListener { public: AccessibilityBridge(FlutterView*, mojo::ServiceProvider*); ~AccessibilityBridge() override; void UpdateSemanticsTree(mojo::Array) override; - AccessibilityNode* UpdateNode(const semantics::SemanticsNodePtr& node); - void RemoveNode(AccessibilityNode* node); - - base::WeakPtr AsWeakPtr(); FlutterView* view() { return view_; } semantics::SemanticsServer* server() { return semantics_server_.get(); } private: - // See class docs above about ownership relationship + SemanticObject* UpdateSemanticObject( + const semantics::SemanticsNodePtr& node, + std::set* updated_objects, + std::set* removed_objects); + void RemoveSemanticObject(SemanticObject* node, + std::set* updated_objects); + FlutterView* view_; semantics::SemanticsServerPtr semantics_server_; - NSMutableDictionary* nodes_; + std::unordered_map objects_; - mojo::StrongBinding binding_; - - base::WeakPtrFactory weak_factory_; + mojo::Binding binding_; DISALLOW_COPY_AND_ASSIGN(AccessibilityBridge); }; diff --git a/sky/shell/platform/ios/framework/Source/accessibility_bridge.mm b/sky/shell/platform/ios/framework/Source/accessibility_bridge.mm index 253f3995939..3fcb1d2081b 100644 --- a/sky/shell/platform/ios/framework/Source/accessibility_bridge.mm +++ b/sky/shell/platform/ios/framework/Source/accessibility_bridge.mm @@ -5,12 +5,14 @@ #include "sky/shell/platform/ios/framework/Source/accessibility_bridge.h" #import +#include +#include "base/logging.h" #include "mojo/public/cpp/application/connect.h" namespace { -static const uint32_t RootNodeId = 0; +constexpr uint32_t kRootNodeId = 0; // Contains better abstractions than the raw Mojo data structure struct Geometry { @@ -22,19 +24,25 @@ struct Geometry { return *this; } - SkMatrix44 transform = - SkMatrix44(SkMatrix44::Identity_Constructor::kIdentity_Constructor); + SkMatrix44 transform = SkMatrix44(SkMatrix44::kIdentity_Constructor); SkRect rect; }; -} // anonymous namespace +} // namespace -@implementation AccessibilityNode { +@implementation SemanticObject { sky::shell::AccessibilityBridge* _bridge; semantics::SemanticFlagsPtr _flags; semantics::SemanticStringsPtr _strings; Geometry _geometry; + bool _canBeTapped; + bool _canBeLongPressed; + bool _canBeScrolledHorizontally; + bool _canBeScrolledVertically; + bool _canBeAdjusted; + + std::vector _children; } #pragma mark - Override base class designated initializers @@ -51,7 +59,7 @@ struct Geometry { - (instancetype)initWithBridge:(sky::shell::AccessibilityBridge*)bridge uid:(uint32_t)uid { DCHECK(bridge != nil) << "bridge must be set"; - DCHECK(uid >= RootNodeId); + DCHECK(uid >= kRootNodeId); self = [super init]; if (self) { @@ -62,53 +70,61 @@ struct Geometry { return self; } -#pragma mark - Semantics node methods +#pragma mark - Semantic object methods -- (void)update:(const semantics::SemanticsNodePtr&)mojoNode { - DCHECK(_uid == mojoNode->id); +- (void)updateWith:(const semantics::SemanticsNodePtr&)node { + DCHECK(_uid == node->id); - if (!mojoNode->flags.is_null()) { - _flags = mojoNode->flags.Pass(); + if (!node->flags.is_null()) { + _flags = node->flags.Pass(); } - if (!mojoNode->strings.is_null()) { - _strings = mojoNode->strings.Pass(); + if (!node->strings.is_null()) { + _strings = node->strings.Pass(); } - if (!mojoNode->geometry.is_null()) { - _geometry = mojoNode->geometry; + if (!node->geometry.is_null()) { + _geometry = node->geometry; } - if (!mojoNode->children.is_null()) { - // Mark existing children as orphans - NSArray* oldChildren = _children; - for (AccessibilityNode* child in oldChildren) { - DCHECK(child->_parent != nil); - child->_parent = nil; - } - - // Set the new list of children - NSMutableArray* children = [[NSMutableArray alloc] init]; - _children = children; - for (const semantics::SemanticsNodePtr& mojoChild : mojoNode->children) { - AccessibilityNode* child = _bridge->UpdateNode(mojoChild); - child->_parent = self; - [children insertObject:child atIndex:0]; - } - - // Remove those children that are still marked as orphans - for (AccessibilityNode* child in oldChildren) { - if (child->_parent == nil) { - _bridge->RemoveNode(child); + if (!node->actions.is_null()) { + _canBeTapped = false; + _canBeLongPressed = false; + _canBeScrolledHorizontally = false; + _canBeScrolledVertically = false; + for (int action : node->actions) { + switch (static_cast(action)) { + case semantics::SemanticAction::TAP: + _canBeTapped = true; + break; + case semantics::SemanticAction::LONG_PRESS: + _canBeLongPressed = true; + break; + case semantics::SemanticAction::SCROLL_LEFT: + case semantics::SemanticAction::SCROLL_RIGHT: + _canBeScrolledHorizontally = true; + break; + case semantics::SemanticAction::SCROLL_UP: + case semantics::SemanticAction::SCROLL_DOWN: + _canBeScrolledVertically = true; + break; + case semantics::SemanticAction::INCREASE: + case semantics::SemanticAction::DECREASE: + _canBeAdjusted = true; + break; } } - [oldChildren release]; } } -- (void)remove { - _parent = nil; - _bridge->RemoveNode(self); +- (std::vector*)children { + return &_children; +} + +- (void)neuter { + _bridge = nullptr; + _children.clear(); + self.parent = nil; } #pragma mark - UIAccessibility overrides @@ -117,27 +133,30 @@ struct Geometry { // Note: hit detection will only apply to elements that report // -isAccessibilityElement of YES. The framework will continue scanning the // entire element tree looking for such a hit. - return (_flags->canBeTapped || _children.count == 0); + return _canBeTapped || _children.empty(); } - (NSString*)accessibilityLabel { - return (_strings.is_null() || _strings->label.get().empty()) - ? nil - : @(_strings->label.data()); + if (_strings.is_null() || _strings->label.get().empty()) { + return nil; + } + return @(_strings->label.data()); } - (UIAccessibilityTraits)accessibilityTraits { UIAccessibilityTraits traits = UIAccessibilityTraitNone; - if (_flags->canBeTapped) + if (_canBeTapped) { traits |= UIAccessibilityTraitButton; - if (_flags->isAdjustable) + } + if (_canBeAdjusted) { traits |= UIAccessibilityTraitAdjustable; + } return traits; } - (CGRect)accessibilityFrame { SkMatrix44 globalTransform = _geometry.transform; - for (AccessibilityNode* parent = _parent; parent; parent = parent.parent) { + for (SemanticObject* parent = _parent; parent; parent = parent.parent) { globalTransform = globalTransform * parent->_geometry.transform; } @@ -159,22 +178,28 @@ struct Geometry { #pragma mark - UIAccessibilityElement protocol - (id)accessibilityContainer { - return (_uid == RootNodeId) ? _bridge->view() : _parent; + return (_uid == kRootNodeId) ? _bridge->view() : _parent; } #pragma mark - UIAccessibilityContainer overrides - (NSInteger)accessibilityElementCount { - return _children.count; + return (NSInteger)_children.size(); } - (nullable id)accessibilityElementAtIndex:(NSInteger)index { - return (index < 0 || index >= (NSInteger)_children.count ? nil - : _children[index]); + if (index < 0 || index >= (NSInteger)_children.size()) { + return nil; + } + return _children[index]; } - (NSInteger)indexOfAccessibilityElement:(id)element { - return (_children == nil) ? NSNotFound : [_children indexOfObject:element]; + auto it = std::find(_children.begin(), _children.end(), element); + if (it == _children.end()) { + return NSNotFound; + } + return it - _children.begin(); } #pragma mark - UIAccessibilityAction overrides @@ -185,11 +210,15 @@ struct Geometry { } - (void)accessibilityIncrement { - // TODO(tvolkert): Implement + if (_canBeAdjusted) { + _bridge->server()->PerformAction(_uid, semantics::SemanticAction::INCREASE); + } } - (void)accessibilityDecrement { - // TODO(tvolkert): Implement + if (_canBeAdjusted) { + _bridge->server()->PerformAction(_uid, semantics::SemanticAction::DECREASE); + } } - (BOOL)accessibilityScroll:(UIAccessibilityScrollDirection)direction { @@ -197,17 +226,18 @@ struct Geometry { switch (direction) { case UIAccessibilityScrollDirectionRight: case UIAccessibilityScrollDirectionLeft: - canBeScrolled = _flags->canBeScrolledHorizontally; + canBeScrolled = _canBeScrolledHorizontally; break; case UIAccessibilityScrollDirectionUp: case UIAccessibilityScrollDirectionDown: - canBeScrolled = _flags->canBeScrolledVertically; + canBeScrolled = _canBeScrolledVertically; break; default: // Note: page turning of reading content is not currently supported // (UIAccessibilityScrollDirectionNext, // UIAccessibilityScrollDirectionPrevious) canBeScrolled = NO; + break; } if (!canBeScrolled) { @@ -216,16 +246,16 @@ struct Geometry { switch (direction) { case UIAccessibilityScrollDirectionRight: - _bridge->server()->ScrollRight(_uid); + _bridge->server()->PerformAction(_uid, semantics::SemanticAction::SCROLL_RIGHT); break; case UIAccessibilityScrollDirectionLeft: - _bridge->server()->ScrollLeft(_uid); + _bridge->server()->PerformAction(_uid, semantics::SemanticAction::SCROLL_LEFT); break; case UIAccessibilityScrollDirectionUp: - _bridge->server()->ScrollDown(_uid); + _bridge->server()->PerformAction(_uid, semantics::SemanticAction::SCROLL_UP); break; case UIAccessibilityScrollDirectionDown: - _bridge->server()->ScrollUp(_uid); + _bridge->server()->PerformAction(_uid, semantics::SemanticAction::SCROLL_DOWN); break; default: DCHECK(false) << "Unsupported scroll direction: " << direction; @@ -246,13 +276,6 @@ struct Geometry { return NO; } -#pragma mark - Misc - -- (void)dealloc { - [_children release]; - [super dealloc]; -} - @end #pragma mark - AccessibilityBridge impl @@ -262,59 +285,83 @@ namespace shell { AccessibilityBridge::AccessibilityBridge(FlutterView* view, mojo::ServiceProvider* serviceProvider) - : view_(view), binding_(this), weak_factory_(this) { + : view_(view), binding_(this) { mojo::ConnectToService(serviceProvider, mojo::GetProxy(&semantics_server_)); mojo::InterfaceHandle listener; binding_.Bind(&listener); semantics_server_->AddSemanticsListener(listener.Pass()); - - nodes_ = [[NSMutableDictionary alloc] init]; -} - -void AccessibilityBridge::UpdateSemanticsTree( - mojo::Array mojoNodes) { - for (const semantics::SemanticsNodePtr& mojoNode : mojoNodes) { - auto node = UpdateNode(mojoNode); - if (mojoNode->id == RootNodeId && view_.accessibilityElements == nil) { - view_.accessibilityElements = @[ node ]; - } - } - DCHECK(view_.accessibilityElements != nil); - - UIAccessibilityPostNotification(UIAccessibilityLayoutChangedNotification, - nil); -} - -base::WeakPtr AccessibilityBridge::AsWeakPtr() { - return weak_factory_.GetWeakPtr(); -} - -AccessibilityNode* AccessibilityBridge::UpdateNode( - const semantics::SemanticsNodePtr& mojoNode) { - AccessibilityNode* node = nodes_[@(mojoNode->id)]; - if (node == nil) { - node = [[AccessibilityNode alloc] initWithBridge:this uid:mojoNode->id]; - DCHECK(nodes_ != nil); - nodes_[@(mojoNode->id)] = node; - [node release]; - } - [node update:mojoNode]; - return node; -} - -void AccessibilityBridge::RemoveNode(AccessibilityNode* node) { - [node retain]; - DCHECK(nodes_[@(node.uid)] != nil); - DCHECK(nodes_[@(node.uid)].parent == nil); - [nodes_ removeObjectForKey:@(node.uid)]; - for (AccessibilityNode* child in node.children) { - [child remove]; - } - [node release]; } AccessibilityBridge::~AccessibilityBridge() { - [nodes_ release]; + for (const auto& entry : objects_) { + SemanticObject* object = entry.second; + [object neuter]; + [object release]; + } +} + +void AccessibilityBridge::UpdateSemanticsTree( + mojo::Array nodes) { + std::set updated_objects; + std::set removed_objects; + + for (const semantics::SemanticsNodePtr& node : nodes) { + UpdateSemanticObject(node, &updated_objects, &removed_objects); + } + + for (SemanticObject* object : removed_objects) { + if (!updated_objects.count(object)) { + RemoveSemanticObject(object, &updated_objects); + } + } + + if (!view_.accessibilityElements) { + view_.accessibilityElements = @[ objects_[kRootNodeId] ]; + } + UIAccessibilityPostNotification( + UIAccessibilityLayoutChangedNotification, nil); +} + +SemanticObject* AccessibilityBridge::UpdateSemanticObject( + const semantics::SemanticsNodePtr& node, + std::set* updated_objects, + std::set* removed_objects) { + SemanticObject* object = objects_[node->id]; + if (!object) { + object = [[SemanticObject alloc] initWithBridge:this uid:node->id]; + objects_[node->id] = object; + } + [object updateWith:node]; + updated_objects->insert(object); + if (!node->children.is_null()) { + std::vector* children = [object children]; + removed_objects->insert(children->begin(), children->end()); + children->clear(); + children->reserve(node->children.size()); + for (const auto& child_node : node->children) { + SemanticObject* child_object = + UpdateSemanticObject(child_node, updated_objects, removed_objects); + child_object.parent = object; + children->push_back(child_object); + } + } + return object; +} + +void AccessibilityBridge::RemoveSemanticObject( + SemanticObject* object, + std::set* updated_objects) { + DCHECK(objects_[object.uid] == object); + objects_.erase(object.uid); + for (SemanticObject* child : *[object children]) { + if (!updated_objects->count(child)) { + DCHECK(child.parent == object); + child.parent = nil; + RemoveSemanticObject(child, updated_objects); + } + } + [object neuter]; + [object release]; } } // namespace shell