Flutter<->Android accessibility bridge

This is another step towards enabling accessibility for Flutter on
Android. It exposes the semantics tree to Android's accessibility API
when accessibility is enabled.

It does not yet:
 - allow one to actually interact with the application via the
   accessibility API
 - expose the accessibility tree to touch exploration
 - implement the accessibility focus API

However, you can see the tree if you run uiautomatorviewer. It is there,
and it matches the UI. At least in Stocks. I didn't test anything else.

sky/shell/BUILD.gn

@@ -57,6 +57,7 @@ source_set("common") {
     "//sky/services/platform",
     "//sky/services/pointer:interfaces",
     "//sky/services/rasterizer:interfaces",
+    "//sky/services/semantics:interfaces",
     "//sky/shell/dart",
     "//ui/gfx",
   ]
@@ -154,6 +155,7 @@ if (is_android) {
 
   android_library("java") {
     java_files = [
+      "platform/android/org/domokit/sky/shell/FlutterSemanticsToAndroidAccessibilityBridge.java",
       "platform/android/org/domokit/sky/shell/PlatformServiceProvider.java",
       "platform/android/org/domokit/sky/shell/PlatformViewAndroid.java",
       "platform/android/org/domokit/sky/shell/ResourceCleaner.java",
@@ -191,6 +193,7 @@ if (is_android) {
       "//sky/services/pointer:interfaces_java",
       "//sky/services/raw_keyboard:interfaces_java",
       "//sky/services/raw_keyboard:raw_keyboard_lib",
+      "//sky/services/semantics:interfaces_java",
       "//sky/services/updater:interfaces_java",
       "//sky/services/vsync:vsync_lib",
     ]

sky/shell/platform/android/org/domokit/sky/shell/FlutterSemanticsToAndroidAccessibilityBridge.java

@@ -0,0 +1,279 @@
+// Copyright 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package org.domokit.sky.shell;
+
+import android.graphics.Rect;
+import android.opengl.Matrix;
+import android.view.View;
+import android.view.accessibility.AccessibilityManager;
+import android.view.accessibility.AccessibilityNodeInfo;
+import android.view.accessibility.AccessibilityNodeProvider;
+
+import org.chromium.mojo.system.MojoException;
+import org.chromium.mojom.semantics.SemanticsListener;
+import org.chromium.mojom.semantics.SemanticsNode;
+import org.chromium.mojom.semantics.SemanticsServer;
+import org.chromium.mojom.sky.ViewportMetrics;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class FlutterSemanticsToAndroidAccessibilityBridge extends AccessibilityNodeProvider
+                                                          implements SemanticsListener {
+    private Map<Integer, PersistentAccessibilityNode> mTreeNodes;
+    private PlatformViewAndroid mOwner;
+
+    FlutterSemanticsToAndroidAccessibilityBridge(PlatformViewAndroid view) {
+        mOwner = view;
+        mTreeNodes = new HashMap<Integer, PersistentAccessibilityNode>();
+    }
+
+    @Override
+    public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId) {
+
+        if (virtualViewId == View.NO_ID) {
+            AccessibilityNodeInfo result = AccessibilityNodeInfo.obtain(mOwner);
+            mOwner.onInitializeAccessibilityNodeInfo(result);
+            if (mTreeNodes.containsKey(0))
+                result.addChild(mOwner, 0);
+            return result;
+        }
+
+        PersistentAccessibilityNode node = mTreeNodes.get(virtualViewId);
+        if (node == null)
+            return null;
+
+        AccessibilityNodeInfo result = AccessibilityNodeInfo.obtain(mOwner, virtualViewId);
+        result.setPackageName(mOwner.getContext().getPackageName());
+        result.setClassName("Flutter"); // Prettier than the more conventional node.getClass().getName()
+        result.setSource(mOwner, virtualViewId);
+
+        if (node.parent != null) {
+            assert node.id > 0;
+            result.setParent(mOwner, node.parent.id);
+        } else {
+            assert node.id == 0;
+            result.setParent(mOwner);
+        }
+
+        Rect bounds = node.getGlobalRect();
+        if (node.parent != null) {
+            Rect parentBounds = node.parent.getGlobalRect();
+            Rect boundsInParent = new Rect(bounds);
+            boundsInParent.offset(-parentBounds.left, -parentBounds.top);
+            result.setBoundsInParent(boundsInParent);
+        } else {
+            result.setBoundsInParent(bounds);
+        }
+        result.setBoundsInScreen(bounds);
+        result.setVisibleToUser(true);
+
+        // TODO(ianh): Add support for interactivity:
+        // private boolean canBeTapped;
+        // private boolean canBeLongPressed;
+        // private boolean canBeScrolledHorizontally;
+        // private boolean canBeScrolledVertically;
+
+        result.setCheckable(node.hasCheckedState);
+        result.setChecked(node.isChecked);
+        result.setText(node.label);
+
+        for (PersistentAccessibilityNode child : node.children) {
+            result.addChild(mOwner, child.id);
+        }
+
+        return result;
+    }
+
+    @Override
+    public void updateSemanticsTree(SemanticsNode[] nodes) {
+        for (SemanticsNode node : nodes) {
+            updateSemanticsNode(node);
+        }
+    }
+
+    private PersistentAccessibilityNode updateSemanticsNode(SemanticsNode node) {
+        PersistentAccessibilityNode persistentNode = mTreeNodes.get(node.id);
+        if (persistentNode != null) {
+            persistentNode.update(node);
+        } else {
+            persistentNode = new PersistentAccessibilityNode(node);
+            mTreeNodes.put(node.id, persistentNode);
+        }
+        assert persistentNode != null;
+        return persistentNode;
+    }
+
+    public void removePersistentNode(PersistentAccessibilityNode node) {
+        assert mTreeNodes.containsKey(node.id);
+        assert mTreeNodes.get(node.id).parent == null;
+        mTreeNodes.remove(node.id);
+    }
+
+    public void reset() {
+        mTreeNodes.clear();
+    }
+
+    private class PersistentAccessibilityNode {
+        PersistentAccessibilityNode(SemanticsNode node) {
+            update(node);
+        }
+        void update(SemanticsNode node) {
+            if (id == -1) {
+                id = node.id;
+                assert node.flags != null;
+                assert node.strings != null;
+                assert node.geometry != null;
+                assert node.children != null;
+            }
+            assert id == node.id;
+            if (node.flags != null) {
+                canBeTapped = node.flags.canBeTapped;
+                canBeLongPressed = node.flags.canBeLongPressed;
+                canBeScrolledHorizontally = node.flags.canBeScrolledHorizontally;
+                canBeScrolledVertically = node.flags.canBeScrolledVertically;
+                hasCheckedState = node.flags.hasCheckedState;
+                isChecked = node.flags.isChecked;
+            }
+            if (node.strings != null) {
+                label = node.strings.label;
+            }
+            if (node.geometry != null) {
+                transform = node.geometry.transform;
+                left = node.geometry.left;
+                top = node.geometry.top;
+                width = node.geometry.width;
+                height = node.geometry.height;
+            }
+            if (node.children != null) {
+                List<PersistentAccessibilityNode> oldChildren = children;
+                children = new ArrayList<PersistentAccessibilityNode>(node.children.length);
+                if (oldChildren != null) {
+                    for (PersistentAccessibilityNode child : oldChildren) {
+                        assert child.parent != null;
+                        child.parent = null;
+                    }
+                }
+                for (SemanticsNode childNode : node.children) {
+                    PersistentAccessibilityNode child = FlutterSemanticsToAndroidAccessibilityBridge.this.updateSemanticsNode(childNode);
+                    assert child != null;
+                    child.parent = this;
+                    children.add(child);
+                }
+                if (oldChildren != null) {
+                    for (PersistentAccessibilityNode child : oldChildren) {
+                        if (child.parent == null) {
+                            FlutterSemanticsToAndroidAccessibilityBridge.this.removePersistentNode(child);
+                        }
+                    }
+                }
+            }
+            if (node.geometry != null) {
+                // has to be done after children are updated
+                // since they also get marked dirty
+                invalidateGlobalGeometry();
+            }
+            // TODO(ianh): Notify Android that our tree is dirty
+        }
+
+        // fields that we pass straight to the Android accessibility API
+        public int id = -1;
+        public PersistentAccessibilityNode parent;
+        public boolean canBeTapped;
+        public boolean canBeLongPressed;
+        public boolean canBeScrolledHorizontally;
+        public boolean canBeScrolledVertically;
+        public boolean hasCheckedState;
+        public boolean isChecked;
+        public String label;
+        public List<PersistentAccessibilityNode> children;
+
+        // geometry, which we have to convert to global coordinates to send to Android
+        private float[] transform; // can be null, meaning identity transform
+        private float left;
+        private float top;
+        private float width;
+        private float height;
+
+        private boolean geometryDirty = true;
+        private void invalidateGlobalGeometry() {
+            if (geometryDirty) {
+                return;
+            }
+            geometryDirty = true;
+            for (PersistentAccessibilityNode child : children) {
+                child.invalidateGlobalGeometry();
+            }
+        }
+
+        private float[] globalTransform; // cached transform from the root node to this node
+        private Rect globalRect; // cached Rect of bounds of this node in coordinate space of the root node
+
+        private float[] getGlobalTransform() {
+            if (geometryDirty) {
+                if (parent == null) {
+                    globalTransform = transform;
+                } else {
+                    float[] parentTransform = parent.getGlobalTransform();
+                    if (transform == null) {
+                        globalTransform = parentTransform;
+                    } else if (parentTransform == null) {
+                        globalTransform = transform;
+                    } else {
+                        globalTransform = new float[16];
+                        Matrix.multiplyMM(globalTransform, 0, transform, 0, parentTransform, 0);
+                    }
+                }
+            }
+            return globalTransform;
+        }
+
+        private float[] transformPoint(float[] transform, float[] point) {
+            if (transform == null)
+                return point; // this is a 4-item array but the caller will ignore all but the first two items
+            float[] transformedPoint = new float[4];
+            Matrix.multiplyMV(transformedPoint, 0, transform, 0, point, 0);
+            assert transformedPoint[2] == 0;
+            return new float[]{transformedPoint[0] / transformedPoint[3],
+                               transformedPoint[1] / transformedPoint[3]};
+        }
+
+        private float min(float a, float b, float c, float d) {
+            return Math.min(a, Math.min(b, Math.min(c, d)));
+        }
+
+        private float max(float a, float b, float c, float d) {
+            return Math.max(a, Math.max(b, Math.max(c, d)));
+        }
+
+        public Rect getGlobalRect() {
+            if (geometryDirty) {
+                float[] transform = getGlobalTransform();
+                float[] point1 = transformPoint(transform, new float[]{left,         top,          0, 1});
+                float[] point2 = transformPoint(transform, new float[]{left + width, top,          0, 1});
+                float[] point3 = transformPoint(transform, new float[]{left + width, top + height, 0, 1});
+                float[] point4 = transformPoint(transform, new float[]{left,         top + height, 0, 1});
+                // TODO(ianh): Scaling here is a hack to work around #1360.
+                float scale = mOwner.getDevicePixelRatio();
+                globalRect = new Rect(
+                  Math.round(min(point1[0], point2[0], point3[0], point4[0]) * scale),
+                  Math.round(min(point1[1], point2[1], point3[1], point4[1]) * scale),
+                  Math.round(max(point1[0], point2[0], point3[0], point4[0]) * scale),
+                  Math.round(max(point1[1], point2[1], point3[1], point4[1]) * scale)
+                );
+            }
+            return globalRect;
+        }
+    }
+
+    @Override
+    public void close() {}
+
+    @Override
+    public void onConnectionError(MojoException e) {}
+
+}

sky/shell/platform/android/org/domokit/sky/shell/PlatformViewAndroid.java

@@ -5,6 +5,8 @@
 package org.domokit.sky.shell;
 
 import android.content.Context;
+import android.opengl.Matrix;
+import android.graphics.Rect;
 import android.os.Build;
 import android.util.Log;
 import android.view.KeyEvent;
@@ -13,16 +15,21 @@ import android.view.Surface;
 import android.view.SurfaceHolder;
 import android.view.SurfaceView;
 import android.view.View;
+import android.view.WindowInsets;
+import android.view.accessibility.AccessibilityManager;
+import android.view.accessibility.AccessibilityNodeInfo;
+import android.view.accessibility.AccessibilityNodeProvider;
 import android.view.inputmethod.EditorInfo;
 import android.view.inputmethod.InputConnection;
-import android.view.WindowInsets;
 
 import org.chromium.base.JNINamespace;
 import org.chromium.mojo.bindings.InterfaceRequest;
 import org.chromium.mojo.system.Core;
-import org.chromium.mojo.system.impl.CoreImpl;
 import org.chromium.mojo.system.MessagePipeHandle;
+import org.chromium.mojo.system.MojoException;
 import org.chromium.mojo.system.Pair;
+import org.chromium.mojo.system.impl.CoreImpl;
+import org.chromium.mojo.system.impl.CoreImpl;
 import org.chromium.mojom.editing.Keyboard;
 import org.chromium.mojom.mojo.ServiceProvider;
 import org.chromium.mojom.pointer.Pointer;
@@ -30,32 +37,43 @@ import org.chromium.mojom.pointer.PointerKind;
 import org.chromium.mojom.pointer.PointerPacket;
 import org.chromium.mojom.pointer.PointerType;
 import org.chromium.mojom.raw_keyboard.RawKeyboardService;
+import org.chromium.mojom.semantics.SemanticsListener;
+import org.chromium.mojom.semantics.SemanticsNode;
+import org.chromium.mojom.semantics.SemanticsServer;
 import org.chromium.mojom.sky.ServicesData;
 import org.chromium.mojom.sky.SkyEngine;
 import org.chromium.mojom.sky.ViewportMetrics;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import org.domokit.editing.KeyboardImpl;
 import org.domokit.editing.KeyboardViewState;
 import org.domokit.raw_keyboard.RawKeyboardServiceImpl;
 import org.domokit.raw_keyboard.RawKeyboardServiceState;
 
+import org.domokit.sky.shell.FlutterSemanticsToAndroidAccessibilityBridge;
+
 /**
  * A view containing Sky
  */
 @JNINamespace("sky::shell")
-public class PlatformViewAndroid extends SurfaceView {
+public class PlatformViewAndroid extends SurfaceView
+  implements AccessibilityManager.AccessibilityStateChangeListener,
+             AccessibilityManager.TouchExplorationStateChangeListener {
     private static final String TAG = "PlatformViewAndroid";
 
     private long mNativePlatformView;
     private SkyEngine.Proxy mSkyEngine;
     private PlatformServiceProvider mServiceProvider;
+    private ServiceProvider.Proxy mDartServiceProvider;
     private final SurfaceHolder.Callback mSurfaceCallback;
     private final ViewportMetrics mMetrics;
     private final KeyboardViewState mKeyboardState;
     private final RawKeyboardServiceState mRawKeyboardState;
+    private final AccessibilityManager mAccessibilityManager;
 
     public PlatformViewAndroid(Context context) {
         super(context);
@@ -89,6 +107,8 @@ public class PlatformViewAndroid extends SurfaceView {
 
         mKeyboardState = new KeyboardViewState(this);
         mRawKeyboardState = new RawKeyboardServiceState();
+
+        mAccessibilityManager = (AccessibilityManager)getContext().getSystemService(Context.ACCESSIBILITY_SERVICE);
     }
 
     @Override
@@ -109,6 +129,10 @@ public class PlatformViewAndroid extends SurfaceView {
         return mSkyEngine;
     }
 
+    float getDevicePixelRatio() {
+        return mMetrics.devicePixelRatio;
+    }
+
     void destroy() {
         getHolder().removeCallback(mSurfaceCallback);
         nativeDetach(mNativePlatformView);
@@ -279,10 +303,17 @@ public class PlatformViewAndroid extends SurfaceView {
                 ServiceProvider.MANAGER.getInterfaceRequest(core);
         ServiceProvider.MANAGER.bind(mServiceProvider, serviceProvider.second);
 
+        Pair<ServiceProvider.Proxy, InterfaceRequest<ServiceProvider>> dartServiceProvider =
+                ServiceProvider.MANAGER.getInterfaceRequest(core);
+        mDartServiceProvider = dartServiceProvider.first;
+
         ServicesData services = new ServicesData();
         services.servicesProvidedByEmbedder = serviceProvider.first;
+        services.servicesProvidedToEmbedder = dartServiceProvider.second;
         mSkyEngine.setServices(services);
 
+        resetAccessibilityTree();
+
         mSkyEngine.runFromBundle(path);
     }
 
@@ -291,4 +322,61 @@ public class PlatformViewAndroid extends SurfaceView {
     private static native void nativeSurfaceCreated(long nativePlatformViewAndroid,
                                                     Surface surface);
     private static native void nativeSurfaceDestroyed(long nativePlatformViewAndroid);
+
+
+    // ACCESSIBILITY
+
+    @Override
+    protected void onAttachedToWindow() {
+        super.onAttachedToWindow();
+        if (mAccessibilityManager.isEnabled() || mAccessibilityManager.isTouchExplorationEnabled())
+          ensureAccessibilityEnabled();
+        mAccessibilityManager.addAccessibilityStateChangeListener(this);
+        mAccessibilityManager.addTouchExplorationStateChangeListener(this);
+    }
+
+    @Override
+    public void onAccessibilityStateChanged(boolean enabled) {
+        if (enabled)
+            ensureAccessibilityEnabled();
+    }
+
+    @Override
+    public void onTouchExplorationStateChanged(boolean enabled) {
+        if (enabled)
+            ensureAccessibilityEnabled();
+        // TODO(ianh): else, actually discard the state for exploration
+    }
+
+    private FlutterSemanticsToAndroidAccessibilityBridge mAccessibilityNodeProvider;
+    private SemanticsServer.Proxy mSemanticsServer;
+
+    void ensureAccessibilityEnabled() {
+        if (mAccessibilityNodeProvider == null) {
+            mAccessibilityNodeProvider = new FlutterSemanticsToAndroidAccessibilityBridge(this);
+            Core core = CoreImpl.getInstance();
+            Pair<SemanticsServer.Proxy, InterfaceRequest<SemanticsServer>> server =
+                      SemanticsServer.MANAGER.getInterfaceRequest(core);
+            mSemanticsServer = server.first;
+            mDartServiceProvider.connectToService(SemanticsServer.MANAGER.getName(), server.second.passHandle());
+            mSemanticsServer.addSemanticsListener(mAccessibilityNodeProvider);
+        }
+        assert mSemanticsServer != null;
+    }
+
+    @Override
+    public AccessibilityNodeProvider getAccessibilityNodeProvider() {
+        ensureAccessibilityEnabled();
+        return mAccessibilityNodeProvider;
+    }
+
+    // TODO(ianh): implement touch exploration
+
+    // TODO(ianh): implement accessibility focus
+
+    void resetAccessibilityTree() {
+        if (mAccessibilityNodeProvider != null)
+            mAccessibilityNodeProvider.reset();
+    }
+
 }

sky/shell/ui/internals.cc

@@ -108,8 +108,11 @@ Internals::Internals(ServicesDataPtr services,
         services_->services_provided_by_embedder.get());
   }
   service_provider_impl_.AddService<mojo::asset_bundle::AssetUnpacker>(this);
-
-  services_provided_to_embedder_ = GetProxy(&services_from_dart_);
+  if (services_ && services_->services_provided_to_embedder.is_pending()) {
+    services_provided_to_embedder_ = services_->services_provided_to_embedder.Pass();
+  } else {
+    services_provided_to_embedder_ = GetProxy(&services_from_dart_);
+  }
 }
 
 Internals::~Internals() {