[a11y] Support TalkBack reading by word, character, and paragraph (flutter/engine#17626)

engine/src/flutter/shell/platform/android/BUILD.gn

@@ -446,6 +446,7 @@ action("robolectric_tests") {
     "test/io/flutter/plugins/GeneratedPluginRegistrant.java",
     "test/io/flutter/util/FakeKeyEvent.java",
     "test/io/flutter/util/PreconditionsTest.java",
+    "test/io/flutter/view/AccessibilityBridgeTest.java",
   ]
 
   outputs = [

engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java

@@ -739,6 +739,12 @@ public class AccessibilityBridge extends AccessibilityNodeProvider {
       result.setLiveRegion(View.ACCESSIBILITY_LIVE_REGION_POLITE);
     }
 
+    if (semanticsNode.hasFlag(Flag.IS_TEXT_FIELD)) {
+      result.setText(semanticsNode.getValueLabelHint());
+    } else if (!semanticsNode.hasFlag(Flag.SCOPES_ROUTE)) {
+      result.setContentDescription(semanticsNode.getValueLabelHint());
+    }
+
     boolean hasCheckedState = semanticsNode.hasFlag(Flag.HAS_CHECKED_STATE);
     boolean hasToggledState = semanticsNode.hasFlag(Flag.HAS_TOGGLED_STATE);
     if (BuildConfig.DEBUG && (hasCheckedState && hasToggledState)) {
@@ -747,7 +753,6 @@ public class AccessibilityBridge extends AccessibilityNodeProvider {
     result.setCheckable(hasCheckedState || hasToggledState);
     if (hasCheckedState) {
       result.setChecked(semanticsNode.hasFlag(Flag.IS_CHECKED));
-      result.setContentDescription(semanticsNode.getValueLabelHint());
       if (semanticsNode.hasFlag(Flag.IS_IN_MUTUALLY_EXCLUSIVE_GROUP)) {
         result.setClassName("android.widget.RadioButton");
       } else {
@@ -756,13 +761,7 @@ public class AccessibilityBridge extends AccessibilityNodeProvider {
     } else if (hasToggledState) {
       result.setChecked(semanticsNode.hasFlag(Flag.IS_TOGGLED));
       result.setClassName("android.widget.Switch");
-      result.setContentDescription(semanticsNode.getValueLabelHint());
-    } else if (!semanticsNode.hasFlag(Flag.SCOPES_ROUTE)) {
-      // Setting the text directly instead of the content description
-      // will replace the "checked" or "not-checked" label.
-      result.setText(semanticsNode.getValueLabelHint());
     }
-
     result.setSelected(semanticsNode.hasFlag(Flag.IS_SELECTED));
 
     // Heading support
@@ -1660,7 +1659,7 @@ public class AccessibilityBridge extends AccessibilityNodeProvider {
 
   // Must match SemanticsFlag in semantics.dart
   // https://github.com/flutter/engine/blob/master/lib/ui/semantics.dart
-  private enum Flag {
+  /* Package */ enum Flag {
     HAS_CHECKED_STATE(1 << 0),
     IS_CHECKED(1 << 1),
     IS_SELECTED(1 << 2),

engine/src/flutter/shell/platform/android/platform_view_android.cc

@@ -247,6 +247,14 @@ void PlatformViewAndroid::UpdateSemantics(
           value.second.customAccessibilityActions.size() * kBytesPerChild;
     }
 
+    // The encoding defined here is used in:
+    //
+    //  * AccessibilityBridge.java
+    //  * AccessibilityBridgeTest.java
+    //  * accessibility_bridge.mm
+    //
+    // If any of the encoding structure or length is changed, those locations
+    // must be updated (at a minimum).
     std::vector<uint8_t> buffer(num_bytes);
     int32_t* buffer_int32 = reinterpret_cast<int32_t*>(&buffer[0]);
     float* buffer_float32 = reinterpret_cast<float*>(&buffer[0]);

engine/src/flutter/shell/platform/android/test/io/flutter/FlutterTestSuite.java

@@ -22,6 +22,7 @@ import io.flutter.plugin.editing.TextInputPluginTest;
 import io.flutter.plugin.platform.PlatformPluginTest;
 import io.flutter.plugin.platform.SingleViewPresentationTest;
 import io.flutter.util.PreconditionsTest;
+import io.flutter.view.AccessibilityBridgeTest;
 import org.junit.runner.RunWith;
 import org.junit.runners.Suite;
 import org.junit.runners.Suite.SuiteClasses;
@@ -55,6 +56,7 @@ import test.io.flutter.embedding.engine.dart.DartExecutorTest;
   SingleViewPresentationTest.class,
   SmokeTest.class,
   TextInputPluginTest.class,
+  AccessibilityBridgeTest.class,
 })
 /** Runs all of the unit tests listed in the {@code @SuiteClasses} annotation. */
 public class FlutterTestSuite {}

engine/src/flutter/shell/platform/android/test/io/flutter/view/AccessibilityBridgeTest.java

@@ -0,0 +1,188 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package io.flutter.view;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import android.content.ContentResolver;
+import android.content.Context;
+import android.view.View;
+import android.view.accessibility.AccessibilityManager;
+import android.view.accessibility.AccessibilityNodeInfo;
+import io.flutter.embedding.engine.systemchannels.AccessibilityChannel;
+import io.flutter.plugin.platform.PlatformViewsAccessibilityDelegate;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.robolectric.RobolectricTestRunner;
+import org.robolectric.annotation.Config;
+
+@Config(manifest = Config.NONE)
+@RunWith(RobolectricTestRunner.class)
+public class AccessibilityBridgeTest {
+
+  @Test
+  public void itDescribesNonTextFieldsWithAContentDescription() {
+    AccessibilityBridge accessibilityBridge = setUpBridge();
+
+    TestSemanticsNode testSemanticsNode = new TestSemanticsNode();
+    testSemanticsNode.label = "Hello, World";
+    TestSemanticsUpdate testSemanticsUpdate = testSemanticsNode.toUpdate();
+
+    accessibilityBridge.updateSemantics(testSemanticsUpdate.buffer, testSemanticsUpdate.strings);
+    AccessibilityNodeInfo nodeInfo = accessibilityBridge.createAccessibilityNodeInfo(0);
+
+    assertEquals(nodeInfo.getContentDescription(), "Hello, World");
+    assertEquals(nodeInfo.getText(), null);
+  }
+
+  @Test
+  public void itDescribesTextFieldsWithText() {
+    AccessibilityBridge accessibilityBridge = setUpBridge();
+
+    TestSemanticsNode testSemanticsNode = new TestSemanticsNode();
+    testSemanticsNode.label = "Hello, World";
+    testSemanticsNode.addFlag(AccessibilityBridge.Flag.IS_TEXT_FIELD);
+    TestSemanticsUpdate testSemanticsUpdate = testSemanticsNode.toUpdate();
+
+    accessibilityBridge.updateSemantics(testSemanticsUpdate.buffer, testSemanticsUpdate.strings);
+    AccessibilityNodeInfo nodeInfo = accessibilityBridge.createAccessibilityNodeInfo(0);
+
+    assertEquals(nodeInfo.getContentDescription(), null);
+    assertEquals(nodeInfo.getText(), "Hello, World");
+  }
+
+  @Test
+  public void itDoesNotContainADescriptionIfScopesRoute() {
+    AccessibilityBridge accessibilityBridge = setUpBridge();
+
+    TestSemanticsNode testSemanticsNode = new TestSemanticsNode();
+    testSemanticsNode.label = "Hello, World";
+    testSemanticsNode.addFlag(AccessibilityBridge.Flag.SCOPES_ROUTE);
+    TestSemanticsUpdate testSemanticsUpdate = testSemanticsNode.toUpdate();
+
+    accessibilityBridge.updateSemantics(testSemanticsUpdate.buffer, testSemanticsUpdate.strings);
+    AccessibilityNodeInfo nodeInfo = accessibilityBridge.createAccessibilityNodeInfo(0);
+
+    assertEquals(nodeInfo.getContentDescription(), null);
+    assertEquals(nodeInfo.getText(), null);
+  }
+
+  AccessibilityBridge setUpBridge() {
+    View view = mock(View.class);
+    Context context = mock(Context.class);
+    when(view.getContext()).thenReturn(context);
+    when(context.getPackageName()).thenReturn("test");
+    AccessibilityChannel accessibilityChannel = mock(AccessibilityChannel.class);
+    AccessibilityManager accessibilityManager = mock(AccessibilityManager.class);
+    ContentResolver contentResolver = mock(ContentResolver.class);
+    PlatformViewsAccessibilityDelegate platformViewsAccessibilityDelegate =
+        mock(PlatformViewsAccessibilityDelegate.class);
+    AccessibilityBridge accessibilityBridge =
+        new AccessibilityBridge(
+            view,
+            accessibilityChannel,
+            accessibilityManager,
+            contentResolver,
+            platformViewsAccessibilityDelegate);
+    return accessibilityBridge;
+  }
+
+  /// The encoding for semantics is described in platform_view_android.cc
+  class TestSemanticsUpdate {
+    TestSemanticsUpdate(ByteBuffer buffer, String[] strings) {
+      this.buffer = buffer;
+      this.strings = strings;
+    }
+
+    final ByteBuffer buffer;
+    final String[] strings;
+  }
+
+  class TestSemanticsNode {
+    TestSemanticsNode() {}
+
+    void addFlag(AccessibilityBridge.Flag flag) {
+      flags |= flag.value;
+    }
+
+    // These fields are declared in the order they should be
+    // encoded.
+    int id = 0;
+    int flags = 0;
+    int actions = 0;
+    int maxValueLength = 0;
+    int currentValueLength = 0;
+    int textSelectionBase = 0;
+    int textSelectionExtent = 0;
+    int platformViewId = -1;
+    int scrollChildren = 0;
+    int scrollIndex = 0;
+    float scrollPosition = 0.0f;
+    float scrollExtentMax = 0.0f;
+    float scrollExtentMin = 0.0f;
+    String label = null;
+    String value = null;
+    String increasedValue = null;
+    String decreasedValue = null;
+    String hint = null;
+    int textDirection = 0;
+    float left = 0.0f;
+    float top = 0.0f;
+    float right = 0.0f;
+    float bottom = 0.0f;
+    // children and custom actions not supported.
+
+    TestSemanticsUpdate toUpdate() {
+      ArrayList<String> strings = new ArrayList<String>();
+      ByteBuffer bytes = ByteBuffer.allocate(1000);
+      bytes.putInt(id);
+      bytes.putInt(flags);
+      bytes.putInt(actions);
+      bytes.putInt(maxValueLength);
+      bytes.putInt(currentValueLength);
+      bytes.putInt(textSelectionBase);
+      bytes.putInt(textSelectionExtent);
+      bytes.putInt(platformViewId);
+      bytes.putInt(scrollChildren);
+      bytes.putInt(scrollIndex);
+      bytes.putFloat(scrollPosition);
+      bytes.putFloat(scrollExtentMax);
+      bytes.putFloat(scrollExtentMin);
+      updateString(label, bytes, strings);
+      updateString(value, bytes, strings);
+      updateString(increasedValue, bytes, strings);
+      updateString(decreasedValue, bytes, strings);
+      updateString(hint, bytes, strings);
+      bytes.putInt(textDirection);
+      bytes.putFloat(left);
+      bytes.putFloat(top);
+      bytes.putFloat(right);
+      bytes.putFloat(bottom);
+      // transform.
+      for (int i = 0; i < 16; i++) {
+        bytes.putFloat(0);
+      }
+      // children in traversal order.
+      bytes.putInt(0);
+      // custom actions
+      bytes.putInt(0);
+      bytes.flip();
+      return new TestSemanticsUpdate(bytes, strings.toArray(new String[strings.size()]));
+    }
+  }
+
+  static void updateString(String value, ByteBuffer bytes, ArrayList<String> strings) {
+    if (value == null) {
+      bytes.putInt(-1);
+    } else {
+      strings.add(value);
+      bytes.putInt(strings.size() - 1);
+    }
+  }
+}