diff --git a/sky/shell/BUILD.gn b/sky/shell/BUILD.gn index a48f6d7cd18..cafaad996da 100644 --- a/sky/shell/BUILD.gn +++ b/sky/shell/BUILD.gn @@ -57,6 +57,7 @@ source_set("common") { "//sky/services/platform", "//sky/services/pointer:interfaces", "//sky/services/rasterizer:interfaces", + "//sky/services/semantics:interfaces", "//sky/shell/dart", "//ui/gfx", ] @@ -154,6 +155,7 @@ if (is_android) { android_library("java") { java_files = [ + "platform/android/org/domokit/sky/shell/FlutterSemanticsToAndroidAccessibilityBridge.java", "platform/android/org/domokit/sky/shell/PlatformServiceProvider.java", "platform/android/org/domokit/sky/shell/PlatformViewAndroid.java", "platform/android/org/domokit/sky/shell/ResourceCleaner.java", @@ -191,6 +193,7 @@ if (is_android) { "//sky/services/pointer:interfaces_java", "//sky/services/raw_keyboard:interfaces_java", "//sky/services/raw_keyboard:raw_keyboard_lib", + "//sky/services/semantics:interfaces_java", "//sky/services/updater:interfaces_java", "//sky/services/vsync:vsync_lib", ] diff --git a/sky/shell/platform/android/org/domokit/sky/shell/FlutterSemanticsToAndroidAccessibilityBridge.java b/sky/shell/platform/android/org/domokit/sky/shell/FlutterSemanticsToAndroidAccessibilityBridge.java new file mode 100644 index 00000000000..dae52a6dee0 --- /dev/null +++ b/sky/shell/platform/android/org/domokit/sky/shell/FlutterSemanticsToAndroidAccessibilityBridge.java @@ -0,0 +1,279 @@ +// Copyright 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.domokit.sky.shell; + +import android.graphics.Rect; +import android.opengl.Matrix; +import android.view.View; +import android.view.accessibility.AccessibilityManager; +import android.view.accessibility.AccessibilityNodeInfo; +import android.view.accessibility.AccessibilityNodeProvider; + +import org.chromium.mojo.system.MojoException; +import org.chromium.mojom.semantics.SemanticsListener; +import org.chromium.mojom.semantics.SemanticsNode; +import org.chromium.mojom.semantics.SemanticsServer; +import org.chromium.mojom.sky.ViewportMetrics; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class FlutterSemanticsToAndroidAccessibilityBridge extends AccessibilityNodeProvider + implements SemanticsListener { + private Map mTreeNodes; + private PlatformViewAndroid mOwner; + + FlutterSemanticsToAndroidAccessibilityBridge(PlatformViewAndroid view) { + mOwner = view; + mTreeNodes = new HashMap(); + } + + @Override + public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId) { + + if (virtualViewId == View.NO_ID) { + AccessibilityNodeInfo result = AccessibilityNodeInfo.obtain(mOwner); + mOwner.onInitializeAccessibilityNodeInfo(result); + if (mTreeNodes.containsKey(0)) + result.addChild(mOwner, 0); + return result; + } + + PersistentAccessibilityNode node = mTreeNodes.get(virtualViewId); + if (node == null) + return null; + + AccessibilityNodeInfo result = AccessibilityNodeInfo.obtain(mOwner, virtualViewId); + result.setPackageName(mOwner.getContext().getPackageName()); + result.setClassName("Flutter"); // Prettier than the more conventional node.getClass().getName() + result.setSource(mOwner, virtualViewId); + + if (node.parent != null) { + assert node.id > 0; + result.setParent(mOwner, node.parent.id); + } else { + assert node.id == 0; + result.setParent(mOwner); + } + + Rect bounds = node.getGlobalRect(); + if (node.parent != null) { + Rect parentBounds = node.parent.getGlobalRect(); + Rect boundsInParent = new Rect(bounds); + boundsInParent.offset(-parentBounds.left, -parentBounds.top); + result.setBoundsInParent(boundsInParent); + } else { + result.setBoundsInParent(bounds); + } + result.setBoundsInScreen(bounds); + result.setVisibleToUser(true); + + // TODO(ianh): Add support for interactivity: + // private boolean canBeTapped; + // private boolean canBeLongPressed; + // private boolean canBeScrolledHorizontally; + // private boolean canBeScrolledVertically; + + result.setCheckable(node.hasCheckedState); + result.setChecked(node.isChecked); + result.setText(node.label); + + for (PersistentAccessibilityNode child : node.children) { + result.addChild(mOwner, child.id); + } + + return result; + } + + @Override + public void updateSemanticsTree(SemanticsNode[] nodes) { + for (SemanticsNode node : nodes) { + updateSemanticsNode(node); + } + } + + private PersistentAccessibilityNode updateSemanticsNode(SemanticsNode node) { + PersistentAccessibilityNode persistentNode = mTreeNodes.get(node.id); + if (persistentNode != null) { + persistentNode.update(node); + } else { + persistentNode = new PersistentAccessibilityNode(node); + mTreeNodes.put(node.id, persistentNode); + } + assert persistentNode != null; + return persistentNode; + } + + public void removePersistentNode(PersistentAccessibilityNode node) { + assert mTreeNodes.containsKey(node.id); + assert mTreeNodes.get(node.id).parent == null; + mTreeNodes.remove(node.id); + } + + public void reset() { + mTreeNodes.clear(); + } + + private class PersistentAccessibilityNode { + PersistentAccessibilityNode(SemanticsNode node) { + update(node); + } + void update(SemanticsNode node) { + if (id == -1) { + id = node.id; + assert node.flags != null; + assert node.strings != null; + assert node.geometry != null; + assert node.children != null; + } + assert id == node.id; + if (node.flags != null) { + canBeTapped = node.flags.canBeTapped; + canBeLongPressed = node.flags.canBeLongPressed; + canBeScrolledHorizontally = node.flags.canBeScrolledHorizontally; + canBeScrolledVertically = node.flags.canBeScrolledVertically; + hasCheckedState = node.flags.hasCheckedState; + isChecked = node.flags.isChecked; + } + if (node.strings != null) { + label = node.strings.label; + } + if (node.geometry != null) { + transform = node.geometry.transform; + left = node.geometry.left; + top = node.geometry.top; + width = node.geometry.width; + height = node.geometry.height; + } + if (node.children != null) { + List oldChildren = children; + children = new ArrayList(node.children.length); + if (oldChildren != null) { + for (PersistentAccessibilityNode child : oldChildren) { + assert child.parent != null; + child.parent = null; + } + } + for (SemanticsNode childNode : node.children) { + PersistentAccessibilityNode child = FlutterSemanticsToAndroidAccessibilityBridge.this.updateSemanticsNode(childNode); + assert child != null; + child.parent = this; + children.add(child); + } + if (oldChildren != null) { + for (PersistentAccessibilityNode child : oldChildren) { + if (child.parent == null) { + FlutterSemanticsToAndroidAccessibilityBridge.this.removePersistentNode(child); + } + } + } + } + if (node.geometry != null) { + // has to be done after children are updated + // since they also get marked dirty + invalidateGlobalGeometry(); + } + // TODO(ianh): Notify Android that our tree is dirty + } + + // fields that we pass straight to the Android accessibility API + public int id = -1; + public PersistentAccessibilityNode parent; + public boolean canBeTapped; + public boolean canBeLongPressed; + public boolean canBeScrolledHorizontally; + public boolean canBeScrolledVertically; + public boolean hasCheckedState; + public boolean isChecked; + public String label; + public List children; + + // geometry, which we have to convert to global coordinates to send to Android + private float[] transform; // can be null, meaning identity transform + private float left; + private float top; + private float width; + private float height; + + private boolean geometryDirty = true; + private void invalidateGlobalGeometry() { + if (geometryDirty) { + return; + } + geometryDirty = true; + for (PersistentAccessibilityNode child : children) { + child.invalidateGlobalGeometry(); + } + } + + private float[] globalTransform; // cached transform from the root node to this node + private Rect globalRect; // cached Rect of bounds of this node in coordinate space of the root node + + private float[] getGlobalTransform() { + if (geometryDirty) { + if (parent == null) { + globalTransform = transform; + } else { + float[] parentTransform = parent.getGlobalTransform(); + if (transform == null) { + globalTransform = parentTransform; + } else if (parentTransform == null) { + globalTransform = transform; + } else { + globalTransform = new float[16]; + Matrix.multiplyMM(globalTransform, 0, transform, 0, parentTransform, 0); + } + } + } + return globalTransform; + } + + private float[] transformPoint(float[] transform, float[] point) { + if (transform == null) + return point; // this is a 4-item array but the caller will ignore all but the first two items + float[] transformedPoint = new float[4]; + Matrix.multiplyMV(transformedPoint, 0, transform, 0, point, 0); + assert transformedPoint[2] == 0; + return new float[]{transformedPoint[0] / transformedPoint[3], + transformedPoint[1] / transformedPoint[3]}; + } + + private float min(float a, float b, float c, float d) { + return Math.min(a, Math.min(b, Math.min(c, d))); + } + + private float max(float a, float b, float c, float d) { + return Math.max(a, Math.max(b, Math.max(c, d))); + } + + public Rect getGlobalRect() { + if (geometryDirty) { + float[] transform = getGlobalTransform(); + float[] point1 = transformPoint(transform, new float[]{left, top, 0, 1}); + float[] point2 = transformPoint(transform, new float[]{left + width, top, 0, 1}); + float[] point3 = transformPoint(transform, new float[]{left + width, top + height, 0, 1}); + float[] point4 = transformPoint(transform, new float[]{left, top + height, 0, 1}); + // TODO(ianh): Scaling here is a hack to work around #1360. + float scale = mOwner.getDevicePixelRatio(); + globalRect = new Rect( + Math.round(min(point1[0], point2[0], point3[0], point4[0]) * scale), + Math.round(min(point1[1], point2[1], point3[1], point4[1]) * scale), + Math.round(max(point1[0], point2[0], point3[0], point4[0]) * scale), + Math.round(max(point1[1], point2[1], point3[1], point4[1]) * scale) + ); + } + return globalRect; + } + } + + @Override + public void close() {} + + @Override + public void onConnectionError(MojoException e) {} + +} diff --git a/sky/shell/platform/android/org/domokit/sky/shell/PlatformViewAndroid.java b/sky/shell/platform/android/org/domokit/sky/shell/PlatformViewAndroid.java index 177dc3f3535..f7b40915cd8 100644 --- a/sky/shell/platform/android/org/domokit/sky/shell/PlatformViewAndroid.java +++ b/sky/shell/platform/android/org/domokit/sky/shell/PlatformViewAndroid.java @@ -5,6 +5,8 @@ package org.domokit.sky.shell; import android.content.Context; +import android.opengl.Matrix; +import android.graphics.Rect; import android.os.Build; import android.util.Log; import android.view.KeyEvent; @@ -13,16 +15,21 @@ import android.view.Surface; import android.view.SurfaceHolder; import android.view.SurfaceView; import android.view.View; +import android.view.WindowInsets; +import android.view.accessibility.AccessibilityManager; +import android.view.accessibility.AccessibilityNodeInfo; +import android.view.accessibility.AccessibilityNodeProvider; import android.view.inputmethod.EditorInfo; import android.view.inputmethod.InputConnection; -import android.view.WindowInsets; import org.chromium.base.JNINamespace; import org.chromium.mojo.bindings.InterfaceRequest; import org.chromium.mojo.system.Core; -import org.chromium.mojo.system.impl.CoreImpl; import org.chromium.mojo.system.MessagePipeHandle; +import org.chromium.mojo.system.MojoException; import org.chromium.mojo.system.Pair; +import org.chromium.mojo.system.impl.CoreImpl; +import org.chromium.mojo.system.impl.CoreImpl; import org.chromium.mojom.editing.Keyboard; import org.chromium.mojom.mojo.ServiceProvider; import org.chromium.mojom.pointer.Pointer; @@ -30,32 +37,43 @@ import org.chromium.mojom.pointer.PointerKind; import org.chromium.mojom.pointer.PointerPacket; import org.chromium.mojom.pointer.PointerType; import org.chromium.mojom.raw_keyboard.RawKeyboardService; +import org.chromium.mojom.semantics.SemanticsListener; +import org.chromium.mojom.semantics.SemanticsNode; +import org.chromium.mojom.semantics.SemanticsServer; import org.chromium.mojom.sky.ServicesData; import org.chromium.mojom.sky.SkyEngine; import org.chromium.mojom.sky.ViewportMetrics; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.domokit.editing.KeyboardImpl; import org.domokit.editing.KeyboardViewState; import org.domokit.raw_keyboard.RawKeyboardServiceImpl; import org.domokit.raw_keyboard.RawKeyboardServiceState; +import org.domokit.sky.shell.FlutterSemanticsToAndroidAccessibilityBridge; + /** * A view containing Sky */ @JNINamespace("sky::shell") -public class PlatformViewAndroid extends SurfaceView { +public class PlatformViewAndroid extends SurfaceView + implements AccessibilityManager.AccessibilityStateChangeListener, + AccessibilityManager.TouchExplorationStateChangeListener { private static final String TAG = "PlatformViewAndroid"; private long mNativePlatformView; private SkyEngine.Proxy mSkyEngine; private PlatformServiceProvider mServiceProvider; + private ServiceProvider.Proxy mDartServiceProvider; private final SurfaceHolder.Callback mSurfaceCallback; private final ViewportMetrics mMetrics; private final KeyboardViewState mKeyboardState; private final RawKeyboardServiceState mRawKeyboardState; + private final AccessibilityManager mAccessibilityManager; public PlatformViewAndroid(Context context) { super(context); @@ -89,6 +107,8 @@ public class PlatformViewAndroid extends SurfaceView { mKeyboardState = new KeyboardViewState(this); mRawKeyboardState = new RawKeyboardServiceState(); + + mAccessibilityManager = (AccessibilityManager)getContext().getSystemService(Context.ACCESSIBILITY_SERVICE); } @Override @@ -109,6 +129,10 @@ public class PlatformViewAndroid extends SurfaceView { return mSkyEngine; } + float getDevicePixelRatio() { + return mMetrics.devicePixelRatio; + } + void destroy() { getHolder().removeCallback(mSurfaceCallback); nativeDetach(mNativePlatformView); @@ -279,10 +303,17 @@ public class PlatformViewAndroid extends SurfaceView { ServiceProvider.MANAGER.getInterfaceRequest(core); ServiceProvider.MANAGER.bind(mServiceProvider, serviceProvider.second); + Pair> dartServiceProvider = + ServiceProvider.MANAGER.getInterfaceRequest(core); + mDartServiceProvider = dartServiceProvider.first; + ServicesData services = new ServicesData(); services.servicesProvidedByEmbedder = serviceProvider.first; + services.servicesProvidedToEmbedder = dartServiceProvider.second; mSkyEngine.setServices(services); + resetAccessibilityTree(); + mSkyEngine.runFromBundle(path); } @@ -291,4 +322,61 @@ public class PlatformViewAndroid extends SurfaceView { private static native void nativeSurfaceCreated(long nativePlatformViewAndroid, Surface surface); private static native void nativeSurfaceDestroyed(long nativePlatformViewAndroid); + + + // ACCESSIBILITY + + @Override + protected void onAttachedToWindow() { + super.onAttachedToWindow(); + if (mAccessibilityManager.isEnabled() || mAccessibilityManager.isTouchExplorationEnabled()) + ensureAccessibilityEnabled(); + mAccessibilityManager.addAccessibilityStateChangeListener(this); + mAccessibilityManager.addTouchExplorationStateChangeListener(this); + } + + @Override + public void onAccessibilityStateChanged(boolean enabled) { + if (enabled) + ensureAccessibilityEnabled(); + } + + @Override + public void onTouchExplorationStateChanged(boolean enabled) { + if (enabled) + ensureAccessibilityEnabled(); + // TODO(ianh): else, actually discard the state for exploration + } + + private FlutterSemanticsToAndroidAccessibilityBridge mAccessibilityNodeProvider; + private SemanticsServer.Proxy mSemanticsServer; + + void ensureAccessibilityEnabled() { + if (mAccessibilityNodeProvider == null) { + mAccessibilityNodeProvider = new FlutterSemanticsToAndroidAccessibilityBridge(this); + Core core = CoreImpl.getInstance(); + Pair> server = + SemanticsServer.MANAGER.getInterfaceRequest(core); + mSemanticsServer = server.first; + mDartServiceProvider.connectToService(SemanticsServer.MANAGER.getName(), server.second.passHandle()); + mSemanticsServer.addSemanticsListener(mAccessibilityNodeProvider); + } + assert mSemanticsServer != null; + } + + @Override + public AccessibilityNodeProvider getAccessibilityNodeProvider() { + ensureAccessibilityEnabled(); + return mAccessibilityNodeProvider; + } + + // TODO(ianh): implement touch exploration + + // TODO(ianh): implement accessibility focus + + void resetAccessibilityTree() { + if (mAccessibilityNodeProvider != null) + mAccessibilityNodeProvider.reset(); + } + } diff --git a/sky/shell/ui/internals.cc b/sky/shell/ui/internals.cc index 4cc76e4f803..a413594e121 100644 --- a/sky/shell/ui/internals.cc +++ b/sky/shell/ui/internals.cc @@ -108,8 +108,11 @@ Internals::Internals(ServicesDataPtr services, services_->services_provided_by_embedder.get()); } service_provider_impl_.AddService(this); - - services_provided_to_embedder_ = GetProxy(&services_from_dart_); + if (services_ && services_->services_provided_to_embedder.is_pending()) { + services_provided_to_embedder_ = services_->services_provided_to_embedder.Pass(); + } else { + services_provided_to_embedder_ = GetProxy(&services_from_dart_); + } } Internals::~Internals() {