From f5ce9117ee7c50dcae2e5382a8206ca2ddafac60 Mon Sep 17 00:00:00 2001 From: chunhtai <47866232+chunhtai@users.noreply.github.com> Date: Wed, 3 Sep 2025 16:37:31 -0700 Subject: [PATCH] Wires up Android API to set section locale (#173364) related https://github.com/flutter/flutter/issues/99600 what still missing: 1. iOS equivalence 2. set application level locale internal only doc: [go/flutter-semantics-language](http://goto.google.com/flutter-semantics-language) ## Pre-launch Checklist - [ ] I read the [Contributor Guide] and followed the process outlined there for submitting PRs. - [ ] I read the [Tree Hygiene] wiki page, which explains my responsibilities. - [ ] I read and followed the [Flutter Style Guide], including [Features we expect every widget to implement]. - [ ] I signed the [CLA]. - [ ] I listed at least one issue that this PR fixes in the description above. - [ ] I updated/added relevant documentation (doc comments with `///`). - [ ] I added new tests to check the change I am making, or this PR is [test-exempt]. - [ ] I followed the [breaking change policy] and added [Data Driven Fixes] where supported. - [ ] All existing and new tests are passing. If you need help, consider asking for advice on the #hackers-new channel on [Discord]. **Note**: The Flutter team is currently trialing the use of [Gemini Code Assist for GitHub](https://developers.google.com/gemini-code-assist/docs/review-github-code). Comments from the `gemini-code-assist` bot should not be taken as authoritative feedback from the Flutter team. If you find its comments useful you can update your code accordingly, but if you are unsure or disagree with the feedback, please feel free to wait for a Flutter team member's review for guidance on which automated comments should be addressed. [Contributor Guide]: https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#overview [Tree Hygiene]: https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md [test-exempt]: https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#tests [Flutter Style Guide]: https://github.com/flutter/flutter/blob/main/docs/contributing/Style-guide-for-Flutter-repo.md [Features we expect every widget to implement]: https://github.com/flutter/flutter/blob/main/docs/contributing/Style-guide-for-Flutter-repo.md#features-we-expect-every-widget-to-implement [CLA]: https://cla.developers.google.com/ [flutter/tests]: https://github.com/flutter/tests [breaking change policy]: https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#handling-breaking-changes [Discord]: https://github.com/flutter/flutter/blob/main/docs/contributing/Chat.md [Data Driven Fixes]: https://github.com/flutter/flutter/blob/main/docs/contributing/Data-driven-Fixes.md --- .../src/flutter/lib/ui/fixtures/ui_test.dart | 54 +++ engine/src/flutter/lib/ui/semantics.dart | 2 +- .../flutter/lib/ui/semantics/semantics_node.h | 1 + .../ui/semantics/semantics_update_builder.cc | 5 +- .../ui/semantics/semantics_update_builder.h | 4 +- .../semantics_update_builder_unittests.cc | 68 +++- .../flutter/shell/platform/android/BUILD.gn | 1 + .../io/flutter/view/AccessibilityBridge.java | 198 ++++------ .../view/AccessibilityStringBuilder.java | 113 ++++++ .../platform_view_android_delegate.cc | 123 +++--- .../platform_view_android_delegate.h | 4 +- ...latform_view_android_delegate_unittests.cc | 74 +++- .../flutter/view/AccessibilityBridgeTest.java | 374 ++++++++++-------- 13 files changed, 653 insertions(+), 368 deletions(-) create mode 100644 engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityStringBuilder.java diff --git a/engine/src/flutter/lib/ui/fixtures/ui_test.dart b/engine/src/flutter/lib/ui/fixtures/ui_test.dart index 075128636d3..4f0b4a3bfe0 100644 --- a/engine/src/flutter/lib/ui/fixtures/ui_test.dart +++ b/engine/src/flutter/lib/ui/fixtures/ui_test.dart @@ -293,6 +293,60 @@ void sendSemanticsUpdateWithRole() { _semanticsUpdate(builder.build()); } +@pragma('vm:entry-point') +void sendSemanticsUpdateWithLocale() { + final SemanticsUpdateBuilder builder = SemanticsUpdateBuilder(); + + final Float64List transform = Float64List(16); + final Int32List childrenInTraversalOrder = Int32List(0); + final Int32List childrenInHitTestOrder = Int32List(0); + final Int32List additionalActions = Int32List(0); + // Identity matrix 4x4. + transform[0] = 1; + transform[5] = 1; + transform[10] = 1; + builder.updateNode( + id: 0, + flags: SemanticsFlags.none, + actions: 0, + maxValueLength: 0, + currentValueLength: 0, + textSelectionBase: -1, + textSelectionExtent: -1, + platformViewId: -1, + scrollChildren: 0, + scrollIndex: 0, + scrollPosition: 0, + scrollExtentMax: 0, + scrollExtentMin: 0, + rect: Rect.fromLTRB(0, 0, 10, 10), + identifier: "identifier", + label: "label", + labelAttributes: const [], + value: "value", + valueAttributes: const [], + increasedValue: "increasedValue", + increasedValueAttributes: const [], + decreasedValue: "decreasedValue", + decreasedValueAttributes: const [], + hint: "hint", + hintAttributes: const [], + tooltip: "tooltip", + textDirection: TextDirection.ltr, + transform: transform, + childrenInTraversalOrder: childrenInTraversalOrder, + childrenInHitTestOrder: childrenInHitTestOrder, + additionalActions: additionalActions, + headingLevel: 0, + linkUrl: '', + role: SemanticsRole.none, + controlsNodes: null, + inputType: SemanticsInputType.none, + locale: Locale('es', 'MX'), + ); + _semanticsUpdate(builder.build()); +} + @pragma('vm:external-name', 'SemanticsUpdate') external void _semanticsUpdate(SemanticsUpdate update); diff --git a/engine/src/flutter/lib/ui/semantics.dart b/engine/src/flutter/lib/ui/semantics.dart index 4c5f377d12f..e02bcf72477 100644 --- a/engine/src/flutter/lib/ui/semantics.dart +++ b/engine/src/flutter/lib/ui/semantics.dart @@ -1948,7 +1948,7 @@ base class _NativeSemanticsUpdateBuilder extends NativeFieldWrapperClass1 controlsNodes, validationResult.index, inputType.index, - locale?.toString() ?? '', + locale?.toLanguageTag() ?? '', ); } diff --git a/engine/src/flutter/lib/ui/semantics/semantics_node.h b/engine/src/flutter/lib/ui/semantics/semantics_node.h index 7e1ff49fede..602739ac433 100644 --- a/engine/src/flutter/lib/ui/semantics/semantics_node.h +++ b/engine/src/flutter/lib/ui/semantics/semantics_node.h @@ -166,6 +166,7 @@ struct SemanticsNode { std::string linkUrl; SemanticsRole role; SemanticsValidationResult validationResult = SemanticsValidationResult::kNone; + std::string locale; }; // Contains semantic nodes that need to be updated. diff --git a/engine/src/flutter/lib/ui/semantics/semantics_update_builder.cc b/engine/src/flutter/lib/ui/semantics/semantics_update_builder.cc index b461dbe15e2..e349d905bfe 100644 --- a/engine/src/flutter/lib/ui/semantics/semantics_update_builder.cc +++ b/engine/src/flutter/lib/ui/semantics/semantics_update_builder.cc @@ -69,7 +69,9 @@ void SemanticsUpdateBuilder::updateNode( std::string linkUrl, int role, const std::vector& controlsNodes, - int validationResult) { + int validationResult, + int inputType, + std::string locale) { FML_CHECK(scrollChildren == 0 || (scrollChildren > 0 && childrenInHitTestOrder.data())) << "Semantics update contained scrollChildren but did not have " @@ -125,6 +127,7 @@ void SemanticsUpdateBuilder::updateNode( node.role = static_cast(role); node.validationResult = static_cast(validationResult); + node.locale = std::move(locale); nodes_[id] = node; } diff --git a/engine/src/flutter/lib/ui/semantics/semantics_update_builder.h b/engine/src/flutter/lib/ui/semantics/semantics_update_builder.h index f70ef355c0c..40a64c1b12d 100644 --- a/engine/src/flutter/lib/ui/semantics/semantics_update_builder.h +++ b/engine/src/flutter/lib/ui/semantics/semantics_update_builder.h @@ -68,7 +68,9 @@ class SemanticsUpdateBuilder std::string linkUrl, int role, const std::vector& controlsNodes, - int validationResult); + int validationResult, + int inputType, + std::string locale); void updateCustomAction(int id, std::string label, diff --git a/engine/src/flutter/lib/ui/semantics/semantics_update_builder_unittests.cc b/engine/src/flutter/lib/ui/semantics/semantics_update_builder_unittests.cc index 20390c0038b..ce220554f48 100644 --- a/engine/src/flutter/lib/ui/semantics/semantics_update_builder_unittests.cc +++ b/engine/src/flutter/lib/ui/semantics/semantics_update_builder_unittests.cc @@ -23,23 +23,25 @@ TEST_F(SemanticsUpdateBuilderTest, CanHandleAttributedStrings) { ASSERT_FALSE(Dart_IsError(result)); SemanticsUpdate* update = reinterpret_cast(peer); SemanticsNodeUpdates nodes = update->takeNodes(); - ASSERT_EQ(nodes.size(), (size_t)1); - auto node = nodes.find(0)->second; + ASSERT_EQ(nodes.size(), static_cast(1)); + auto found = nodes.find(0); + ASSERT_NE(found, nodes.end()); + SemanticsNode node = found->second; // Should match the updateNode in ui_test.dart. ASSERT_EQ(node.label, "label"); - ASSERT_EQ(node.labelAttributes.size(), (size_t)1); + ASSERT_EQ(node.labelAttributes.size(), static_cast(1)); ASSERT_EQ(node.labelAttributes[0]->start, 1); ASSERT_EQ(node.labelAttributes[0]->end, 2); ASSERT_EQ(node.labelAttributes[0]->type, StringAttributeType::kSpellOut); ASSERT_EQ(node.value, "value"); - ASSERT_EQ(node.valueAttributes.size(), (size_t)1); + ASSERT_EQ(node.valueAttributes.size(), static_cast(1)); ASSERT_EQ(node.valueAttributes[0]->start, 2); ASSERT_EQ(node.valueAttributes[0]->end, 3); ASSERT_EQ(node.valueAttributes[0]->type, StringAttributeType::kSpellOut); ASSERT_EQ(node.hint, "hint"); - ASSERT_EQ(node.hintAttributes.size(), (size_t)1); + ASSERT_EQ(node.hintAttributes.size(), static_cast(1)); ASSERT_EQ(node.hintAttributes[0]->start, 0); ASSERT_EQ(node.hintAttributes[0]->end, 1); ASSERT_EQ(node.hintAttributes[0]->type, StringAttributeType::kLocale); @@ -48,14 +50,14 @@ TEST_F(SemanticsUpdateBuilderTest, CanHandleAttributedStrings) { ASSERT_EQ(local_attribute->locale, "en-MX"); ASSERT_EQ(node.increasedValue, "increasedValue"); - ASSERT_EQ(node.increasedValueAttributes.size(), (size_t)1); + ASSERT_EQ(node.increasedValueAttributes.size(), static_cast(1)); ASSERT_EQ(node.increasedValueAttributes[0]->start, 4); ASSERT_EQ(node.increasedValueAttributes[0]->end, 5); ASSERT_EQ(node.increasedValueAttributes[0]->type, StringAttributeType::kSpellOut); ASSERT_EQ(node.decreasedValue, "decreasedValue"); - ASSERT_EQ(node.decreasedValueAttributes.size(), (size_t)1); + ASSERT_EQ(node.decreasedValueAttributes.size(), static_cast(1)); ASSERT_EQ(node.decreasedValueAttributes[0]->start, 5); ASSERT_EQ(node.decreasedValueAttributes[0]->end, 6); ASSERT_EQ(node.decreasedValueAttributes[0]->type, @@ -99,8 +101,9 @@ TEST_F(SemanticsUpdateBuilderTest, CanHandleSemanticsRole) { ASSERT_FALSE(Dart_IsError(result)); SemanticsUpdate* update = reinterpret_cast(peer); SemanticsNodeUpdates nodes = update->takeNodes(); - ASSERT_EQ(nodes.size(), (size_t)1); - auto node = nodes.find(0)->second; + auto found = nodes.find(0); + ASSERT_NE(found, nodes.end()); + SemanticsNode node = found->second; // Should match the updateNode in ui_test.dart. ASSERT_EQ(node.role, SemanticsRole::kTab); message_latch->Signal(); @@ -123,7 +126,52 @@ TEST_F(SemanticsUpdateBuilderTest, CanHandleSemanticsRole) { auto configuration = RunConfiguration::InferFromSettings(settings); configuration.SetEntrypoint("sendSemanticsUpdateWithRole"); - shell->RunEngine(std::move(configuration), [](auto result) { + shell->RunEngine(std::move(configuration), [](Engine::RunStatus result) { + ASSERT_EQ(result, Engine::RunStatus::Success); + }); + + message_latch->Wait(); + DestroyShell(std::move(shell), task_runners); +} + +TEST_F(SemanticsUpdateBuilderTest, CanHandleSemanticsLocale) { + auto message_latch = std::make_shared(); + + auto nativeSemanticsUpdate = [message_latch](Dart_NativeArguments args) { + auto handle = Dart_GetNativeArgument(args, 0); + intptr_t peer = 0; + Dart_Handle result = Dart_GetNativeInstanceField( + handle, tonic::DartWrappable::kPeerIndex, &peer); + ASSERT_FALSE(Dart_IsError(result)); + SemanticsUpdate* update = reinterpret_cast(peer); + SemanticsNodeUpdates nodes = update->takeNodes(); + ASSERT_EQ(nodes.size(), static_cast(1)); + auto found = nodes.find(0); + ASSERT_NE(found, nodes.end()); + SemanticsNode node = found->second; + // Should match the updateNode in ui_test.dart. + ASSERT_EQ(node.locale, "es-MX"); + message_latch->Signal(); + }; + + Settings settings = CreateSettingsForFixture(); + TaskRunners task_runners("test", // label + GetCurrentTaskRunner(), // platform + CreateNewThread(), // raster + CreateNewThread(), // ui + CreateNewThread() // io + ); + + AddNativeCallback("SemanticsUpdate", + CREATE_NATIVE_ENTRY(nativeSemanticsUpdate)); + + std::unique_ptr shell = CreateShell(settings, task_runners); + + ASSERT_TRUE(shell->IsSetup()); + auto configuration = RunConfiguration::InferFromSettings(settings); + configuration.SetEntrypoint("sendSemanticsUpdateWithLocale"); + + shell->RunEngine(std::move(configuration), [](Engine::RunStatus result) { ASSERT_EQ(result, Engine::RunStatus::Success); }); diff --git a/engine/src/flutter/shell/platform/android/BUILD.gn b/engine/src/flutter/shell/platform/android/BUILD.gn index 26ce21b303c..a2445eda33d 100644 --- a/engine/src/flutter/shell/platform/android/BUILD.gn +++ b/engine/src/flutter/shell/platform/android/BUILD.gn @@ -366,6 +366,7 @@ android_java_sources = [ "io/flutter/util/TraceSection.java", "io/flutter/util/ViewUtils.java", "io/flutter/view/AccessibilityBridge.java", + "io/flutter/view/AccessibilityStringBuilder.java", "io/flutter/view/AccessibilityViewEmbedder.java", "io/flutter/view/FlutterCallbackInformation.java", "io/flutter/view/FlutterRunArguments.java", diff --git a/engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java b/engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java index 01d08f1de2b..554d3ce8ad1 100644 --- a/engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java +++ b/engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java @@ -19,11 +19,7 @@ import android.os.Build; import android.os.Bundle; import android.os.Handler; import android.provider.Settings; -import android.text.SpannableString; import android.text.TextUtils; -import android.text.style.LocaleSpan; -import android.text.style.TtsSpan; -import android.text.style.URLSpan; import android.view.MotionEvent; import android.view.View; import android.view.WindowManager; @@ -41,6 +37,10 @@ import io.flutter.embedding.engine.systemchannels.AccessibilityChannel; import io.flutter.plugin.platform.PlatformViewsAccessibilityDelegate; import io.flutter.util.Predicate; import io.flutter.util.ViewUtils; +import io.flutter.view.AccessibilityStringBuilder.LocaleStringAttribute; +import io.flutter.view.AccessibilityStringBuilder.SpellOutStringAttribute; +import io.flutter.view.AccessibilityStringBuilder.StringAttribute; +import io.flutter.view.AccessibilityStringBuilder.StringAttributeType; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.Charset; @@ -134,6 +134,9 @@ public class AccessibilityBridge extends AccessibilityNodeProvider { /// Value is derived from ACTION_TYPE_MASK in AccessibilityNodeInfo.java private static int FIRST_RESOURCE_ID = 267386881; + /// The index value that indicates no string is specified. + private static int EMPTY_STRING_INDEX = -1; + // Real Android View, which internally holds a Flutter UI. @NonNull private final View rootAccessibilityView; @@ -506,6 +509,54 @@ public class AccessibilityBridge extends AccessibilityNodeProvider { platformViewsAccessibilityDelegate.attachAccessibilityBridge(this); } + private static List getStringAttributesFromBuffer( + @NonNull ByteBuffer buffer, @NonNull ByteBuffer[] stringAttributeArgs) { + final int attributesCount = buffer.getInt(); + if (attributesCount == -1) { + return null; + } + final List result = new ArrayList<>(attributesCount); + for (int i = 0; i < attributesCount; ++i) { + final int start = buffer.getInt(); + final int end = buffer.getInt(); + final StringAttributeType type = StringAttributeType.values()[buffer.getInt()]; + switch (type) { + case SPELLOUT: + { + // Pops the -1 size. + buffer.getInt(); + SpellOutStringAttribute attribute = new SpellOutStringAttribute(); + attribute.start = start; + attribute.end = end; + attribute.type = type; + result.add(attribute); + break; + } + case LOCALE: + { + final int argsIndex = buffer.getInt(); + final ByteBuffer args = stringAttributeArgs[argsIndex]; + LocaleStringAttribute attribute = new LocaleStringAttribute(); + attribute.start = start; + attribute.end = end; + attribute.type = type; + attribute.locale = Charset.forName("UTF-8").decode(args).toString(); + result.add(attribute); + break; + } + default: + break; + } + } + return result; + } + + private static String getStringFromBuffer(@NonNull ByteBuffer buffer, @NonNull String[] strings) { + int stringIndex = buffer.getInt(); + + return stringIndex == EMPTY_STRING_INDEX ? null : strings[stringIndex]; + } + /** * Disconnects any listeners and/or delegates that were initialized in {@code * AccessibilityBridge}'s constructor, or added after. @@ -1615,10 +1666,8 @@ public class AccessibilityBridge extends AccessibilityNodeProvider { int id = buffer.getInt(); CustomAccessibilityAction action = getOrCreateAccessibilityAction(id); action.overrideId = buffer.getInt(); - int stringIndex = buffer.getInt(); - action.label = stringIndex == -1 ? null : strings[stringIndex]; - stringIndex = buffer.getInt(); - action.hint = stringIndex == -1 ? null : strings[stringIndex]; + action.label = getStringFromBuffer(buffer, strings); + action.hint = getStringFromBuffer(buffer, strings); } } @@ -2253,34 +2302,6 @@ public class AccessibilityBridge extends AccessibilityNodeProvider { private String hint; } - // When adding a new StringAttributeType, the classes in these file must be - // updated as well. - // * engine/src/flutter/lib/ui/semantics.dart - // * engine/src/flutter/lib/web_ui/lib/semantics.dart - // * engine/src/flutter/lib/ui/semantics/string_attribute.h - - private enum StringAttributeType { - SPELLOUT, - LOCALE, - URL - } - - private static class StringAttribute { - int start; - int end; - StringAttributeType type; - } - - private static class SpellOutStringAttribute extends StringAttribute {} - - private static class LocaleStringAttribute extends StringAttribute { - String locale; - } - - private static class UrlStringAttribute extends StringAttribute { - String url; - } - /** * Flutter {@code SemanticsNode} represented in Java/Android. * @@ -2335,9 +2356,12 @@ public class AccessibilityBridge extends AccessibilityNodeProvider { // API level >= 28; otherwise, this is attached to the end of content description. @Nullable private String tooltip; - // The Url the widget's points to. + // The Url this node points to. @Nullable private String linkUrl; + // The locale of the content of this node. + @Nullable private String locale; + // The id of the sibling node that is before this node in traversal // order. // @@ -2517,40 +2541,26 @@ public class AccessibilityBridge extends AccessibilityNodeProvider { scrollExtentMax = buffer.getFloat(); scrollExtentMin = buffer.getFloat(); - int stringIndex = buffer.getInt(); - - identifier = stringIndex == -1 ? null : strings[stringIndex]; - stringIndex = buffer.getInt(); - - label = stringIndex == -1 ? null : strings[stringIndex]; + identifier = getStringFromBuffer(buffer, strings); + label = getStringFromBuffer(buffer, strings); labelAttributes = getStringAttributesFromBuffer(buffer, stringAttributeArgs); - stringIndex = buffer.getInt(); - value = stringIndex == -1 ? null : strings[stringIndex]; - + value = getStringFromBuffer(buffer, strings); valueAttributes = getStringAttributesFromBuffer(buffer, stringAttributeArgs); - stringIndex = buffer.getInt(); - increasedValue = stringIndex == -1 ? null : strings[stringIndex]; - + increasedValue = getStringFromBuffer(buffer, strings); increasedValueAttributes = getStringAttributesFromBuffer(buffer, stringAttributeArgs); - stringIndex = buffer.getInt(); - decreasedValue = stringIndex == -1 ? null : strings[stringIndex]; - + decreasedValue = getStringFromBuffer(buffer, strings); decreasedValueAttributes = getStringAttributesFromBuffer(buffer, stringAttributeArgs); - stringIndex = buffer.getInt(); - hint = stringIndex == -1 ? null : strings[stringIndex]; - + hint = getStringFromBuffer(buffer, strings); hintAttributes = getStringAttributesFromBuffer(buffer, stringAttributeArgs); - stringIndex = buffer.getInt(); - tooltip = stringIndex == -1 ? null : strings[stringIndex]; - - stringIndex = buffer.getInt(); - linkUrl = stringIndex == -1 ? null : strings[stringIndex]; + tooltip = getStringFromBuffer(buffer, strings); + linkUrl = getStringFromBuffer(buffer, strings); + locale = getStringFromBuffer(buffer, strings); textDirection = TextDirection.fromInt(buffer.getInt()); @@ -2840,29 +2850,28 @@ public class AccessibilityBridge extends AccessibilityNodeProvider { } private CharSequence getValue() { - return createSpannableString(value, valueAttributes); + return new AccessibilityStringBuilder() + .addString(value) + .addAttributes(valueAttributes) + .addLocale(locale) + .build(); } private CharSequence getLabel() { - List attributes = labelAttributes; - if (linkUrl != null && linkUrl.length() > 0) { - if (attributes == null) { - attributes = new ArrayList(); - } else { - attributes = new ArrayList(attributes); - } - UrlStringAttribute uriStringAttribute = new UrlStringAttribute(); - uriStringAttribute.start = 0; - uriStringAttribute.end = label.length(); - uriStringAttribute.url = linkUrl; - uriStringAttribute.type = StringAttributeType.URL; - attributes.add(uriStringAttribute); - } - return createSpannableString(label, attributes); + return new AccessibilityStringBuilder() + .addString(label) + .addAttributes(labelAttributes) + .addUrl(linkUrl) + .addLocale(locale) + .build(); } private CharSequence getHint() { - return createSpannableString(hint, hintAttributes); + return new AccessibilityStringBuilder() + .addString(hint) + .addAttributes(hintAttributes) + .addLocale(locale) + .build(); } private CharSequence getValueLabelHint() { @@ -2894,41 +2903,6 @@ public class AccessibilityBridge extends AccessibilityNodeProvider { } return result; } - - private SpannableString createSpannableString(String string, List attributes) { - if (string == null) { - return null; - } - final SpannableString spannableString = new SpannableString(string); - if (attributes != null) { - for (StringAttribute attribute : attributes) { - switch (attribute.type) { - case SPELLOUT: - { - final TtsSpan ttsSpan = new TtsSpan.Builder<>(TtsSpan.TYPE_VERBATIM).build(); - spannableString.setSpan(ttsSpan, attribute.start, attribute.end, 0); - break; - } - case LOCALE: - { - LocaleStringAttribute localeAttribute = (LocaleStringAttribute) attribute; - Locale locale = Locale.forLanguageTag(localeAttribute.locale); - final LocaleSpan localeSpan = new LocaleSpan(locale); - spannableString.setSpan(localeSpan, attribute.start, attribute.end, 0); - break; - } - case URL: - { - UrlStringAttribute uriAttribute = (UrlStringAttribute) attribute; - final URLSpan urlSpan = new URLSpan(uriAttribute.url); - spannableString.setSpan(urlSpan, attribute.start, attribute.end, 0); - break; - } - } - } - } - return spannableString; - } } /** diff --git a/engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityStringBuilder.java b/engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityStringBuilder.java new file mode 100644 index 00000000000..f6e0c93a5e9 --- /dev/null +++ b/engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityStringBuilder.java @@ -0,0 +1,113 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package io.flutter.view; + +import android.text.SpannableString; +import android.text.style.LocaleSpan; +import android.text.style.TtsSpan; +import android.text.style.URLSpan; +import java.util.List; +import java.util.Locale; + +/** + * Builds a string with accessibility related string spans. + * + *

Use {@code addLocale} to set the locale and/or {@code addUrl} to set the url for the entire + * string. Uses {@code addAttributes} to add any additional {@code StringAttribute} to the string + */ +public class AccessibilityStringBuilder { + // When adding a new StringAttributeType, the classes in these file must be + // updated as well. + // * engine/src/flutter/lib/ui/semantics.dart + // * engine/src/flutter/lib/web_ui/lib/semantics.dart + // * engine/src/flutter/lib/ui/semantics/string_attribute.h + + public enum StringAttributeType { + SPELLOUT, + LOCALE, + } + + public static class StringAttribute { + int start; + int end; + StringAttributeType type; + } + + public static class SpellOutStringAttribute extends StringAttribute {} + + public static class LocaleStringAttribute extends StringAttribute { + String locale; + } + + private static class UrlStringAttribute extends StringAttribute { + String url; + } + + AccessibilityStringBuilder() {} + + private String string; + private List attributes; + private String locale; + private String url; + + AccessibilityStringBuilder addString(String string) { + this.string = string; + return this; + } + + AccessibilityStringBuilder addAttributes(List attributes) { + this.attributes = attributes; + return this; + } + + AccessibilityStringBuilder addLocale(String locale) { + this.locale = locale; + return this; + } + + AccessibilityStringBuilder addUrl(String url) { + this.url = url; + return this; + } + + CharSequence build() { + if (string == null) { + return null; + } + final SpannableString spannableString = new SpannableString(string); + if (attributes != null) { + for (StringAttribute attribute : attributes) { + switch (attribute.type) { + case SPELLOUT: + { + final TtsSpan ttsSpan = new TtsSpan.Builder<>(TtsSpan.TYPE_VERBATIM).build(); + spannableString.setSpan(ttsSpan, attribute.start, attribute.end, 0); + break; + } + case LOCALE: + { + LocaleStringAttribute localeAttribute = (LocaleStringAttribute) attribute; + Locale locale = Locale.forLanguageTag(localeAttribute.locale); + final LocaleSpan localeSpan = new LocaleSpan(locale); + spannableString.setSpan(localeSpan, attribute.start, attribute.end, 0); + break; + } + } + } + } + if (url != null && !url.isEmpty()) { + final URLSpan urlSpan = new URLSpan(url); + spannableString.setSpan(urlSpan, 0, string.length(), 0); + } + + if (locale != null && !locale.isEmpty()) { + Locale localeObject = Locale.forLanguageTag(locale); + final LocaleSpan localeSpan = new LocaleSpan(localeObject); + spannableString.setSpan(localeSpan, 0, string.length(), 0); + } + + return spannableString; + } +} diff --git a/engine/src/flutter/shell/platform/android/platform_view_android_delegate/platform_view_android_delegate.cc b/engine/src/flutter/shell/platform/android/platform_view_android_delegate/platform_view_android_delegate.cc index 120674be04a..b03465bcf5b 100644 --- a/engine/src/flutter/shell/platform/android/platform_view_android_delegate/platform_view_android_delegate.cc +++ b/engine/src/flutter/shell/platform/android/platform_view_android_delegate/platform_view_android_delegate.cc @@ -7,27 +7,27 @@ #include namespace flutter { - +namespace { void putStringAttributesIntoBuffer( const StringAttributes& attributes, - int32_t* buffer_int32, - size_t& position, + int32_t* buffer, + size_t* position, std::vector>& string_attribute_args) { if (attributes.empty()) { - buffer_int32[position++] = -1; + buffer[(*position)++] = PlatformViewAndroidDelegate::kEmptyStringIndex; return; } - buffer_int32[position++] = attributes.size(); + buffer[(*position)++] = attributes.size(); for (const auto& attribute : attributes) { - buffer_int32[position++] = attribute->start; - buffer_int32[position++] = attribute->end; - buffer_int32[position++] = static_cast(attribute->type); + buffer[(*position)++] = attribute->start; + buffer[(*position)++] = attribute->end; + buffer[(*position)++] = static_cast(attribute->type); switch (attribute->type) { case StringAttributeType::kSpellOut: - buffer_int32[position++] = -1; + buffer[(*position)++] = PlatformViewAndroidDelegate::kEmptyStringIndex; break; case StringAttributeType::kLocale: - buffer_int32[position++] = string_attribute_args.size(); + buffer[(*position)++] = string_attribute_args.size(); std::shared_ptr locale_attribute = std::static_pointer_cast(attribute); string_attribute_args.push_back( @@ -37,6 +37,18 @@ void putStringAttributesIntoBuffer( } } +void putStringIntoBuffer(const std::string& string, + int32_t* buffer, + size_t* position, + std::vector& strings) { + if (string.empty()) { + buffer[(*position)++] = PlatformViewAndroidDelegate::kEmptyStringIndex; + } else { + buffer[(*position)++] = strings.size(); + strings.push_back(string); + } +} + int64_t flagsToInt64(flutter::SemanticsFlags flags) { int64_t result = 0; if (flags.isChecked != flutter::SemanticsCheckState::kNone) { @@ -134,6 +146,7 @@ int64_t flagsToInt64(flutter::SemanticsFlags flags) { } return result; } +} // namespace PlatformViewAndroidDelegate::PlatformViewAndroidDelegate( std::shared_ptr jni_facade) @@ -198,73 +211,33 @@ void PlatformViewAndroidDelegate::UpdateSemantics( buffer_float32[position++] = static_cast(node.scrollExtentMax); buffer_float32[position++] = static_cast(node.scrollExtentMin); - if (node.identifier.empty()) { - buffer_int32[position++] = -1; - } else { - buffer_int32[position++] = strings.size(); - strings.push_back(node.identifier); - } - - if (node.label.empty()) { - buffer_int32[position++] = -1; - } else { - buffer_int32[position++] = strings.size(); - strings.push_back(node.label); - } + putStringIntoBuffer(node.identifier, buffer_int32, &position, strings); + putStringIntoBuffer(node.label, buffer_int32, &position, strings); putStringAttributesIntoBuffer(node.labelAttributes, buffer_int32, - position, string_attribute_args); - if (node.value.empty()) { - buffer_int32[position++] = -1; - } else { - buffer_int32[position++] = strings.size(); - strings.push_back(node.value); - } + &position, string_attribute_args); + putStringIntoBuffer(node.value, buffer_int32, &position, strings); putStringAttributesIntoBuffer(node.valueAttributes, buffer_int32, - position, string_attribute_args); - if (node.increasedValue.empty()) { - buffer_int32[position++] = -1; - } else { - buffer_int32[position++] = strings.size(); - strings.push_back(node.increasedValue); - } + &position, string_attribute_args); + putStringIntoBuffer(node.increasedValue, buffer_int32, &position, + strings); putStringAttributesIntoBuffer(node.increasedValueAttributes, buffer_int32, - position, string_attribute_args); - if (node.decreasedValue.empty()) { - buffer_int32[position++] = -1; - } else { - buffer_int32[position++] = strings.size(); - strings.push_back(node.decreasedValue); - } + &position, string_attribute_args); + putStringIntoBuffer(node.decreasedValue, buffer_int32, &position, + strings); putStringAttributesIntoBuffer(node.decreasedValueAttributes, buffer_int32, - position, string_attribute_args); + &position, string_attribute_args); - if (node.hint.empty()) { - buffer_int32[position++] = -1; - } else { - buffer_int32[position++] = strings.size(); - strings.push_back(node.hint); - } + putStringIntoBuffer(node.hint, buffer_int32, &position, strings); + putStringAttributesIntoBuffer(node.hintAttributes, buffer_int32, + &position, string_attribute_args); - putStringAttributesIntoBuffer(node.hintAttributes, buffer_int32, position, - string_attribute_args); - - if (node.tooltip.empty()) { - buffer_int32[position++] = -1; - } else { - buffer_int32[position++] = strings.size(); - strings.push_back(node.tooltip); - } - - if (node.linkUrl.empty()) { - buffer_int32[position++] = -1; - } else { - buffer_int32[position++] = strings.size(); - strings.push_back(node.linkUrl); - } + putStringIntoBuffer(node.tooltip, buffer_int32, &position, strings); + putStringIntoBuffer(node.linkUrl, buffer_int32, &position, strings); + putStringIntoBuffer(node.locale, buffer_int32, &position, strings); buffer_int32[position++] = node.textDirection; buffer_float32[position++] = node.rect.left(); @@ -304,18 +277,10 @@ void PlatformViewAndroidDelegate::UpdateSemantics( const flutter::CustomAccessibilityAction& action = value.second; actions_buffer_int32[actions_position++] = action.id; actions_buffer_int32[actions_position++] = action.overrideId; - if (action.label.empty()) { - actions_buffer_int32[actions_position++] = -1; - } else { - actions_buffer_int32[actions_position++] = action_strings.size(); - action_strings.push_back(action.label); - } - if (action.hint.empty()) { - actions_buffer_int32[actions_position++] = -1; - } else { - actions_buffer_int32[actions_position++] = action_strings.size(); - action_strings.push_back(action.hint); - } + putStringIntoBuffer(action.label, actions_buffer_int32, &actions_position, + action_strings); + putStringIntoBuffer(action.hint, actions_buffer_int32, &actions_position, + action_strings); } // Calling NewDirectByteBuffer in API level 22 and below with a size of zero diff --git a/engine/src/flutter/shell/platform/android/platform_view_android_delegate/platform_view_android_delegate.h b/engine/src/flutter/shell/platform/android/platform_view_android_delegate/platform_view_android_delegate.h index 67304e9d513..2ccbbae5a0b 100644 --- a/engine/src/flutter/shell/platform/android/platform_view_android_delegate/platform_view_android_delegate.h +++ b/engine/src/flutter/shell/platform/android/platform_view_android_delegate/platform_view_android_delegate.h @@ -16,11 +16,13 @@ namespace flutter { class PlatformViewAndroidDelegate { public: - static constexpr size_t kBytesPerNode = 50 * sizeof(int32_t); + static constexpr size_t kBytesPerNode = + 51 * sizeof(int32_t); // The # fields in SemanticsNode static constexpr size_t kBytesPerChild = sizeof(int32_t); static constexpr size_t kBytesPerCustomAction = sizeof(int32_t); static constexpr size_t kBytesPerAction = 4 * sizeof(int32_t); static constexpr size_t kBytesPerStringAttribute = 4 * sizeof(int32_t); + static constexpr int kEmptyStringIndex = -1; explicit PlatformViewAndroidDelegate( std::shared_ptr jni_facade); void UpdateSemantics( diff --git a/engine/src/flutter/shell/platform/android/platform_view_android_delegate/platform_view_android_delegate_unittests.cc b/engine/src/flutter/shell/platform/android/platform_view_android_delegate/platform_view_android_delegate_unittests.cc index e047380faca..75f4097a178 100644 --- a/engine/src/flutter/shell/platform/android/platform_view_android_delegate/platform_view_android_delegate_unittests.cc +++ b/engine/src/flutter/shell/platform/android/platform_view_android_delegate/platform_view_android_delegate_unittests.cc @@ -60,6 +60,7 @@ TEST(PlatformViewShell, UpdateSemanticsDoesFlutterViewUpdateSemantics) { buffer_int32[position++] = expected_strings.size(); // node0.tooltip expected_strings.push_back(node0.tooltip); buffer_int32[position++] = -1; // node0.linkUrl + buffer_int32[position++] = -1; // node0.locale buffer_int32[position++] = node0.textDirection; buffer_float32[position++] = node0.rect.left(); buffer_float32[position++] = node0.rect.top(); @@ -77,7 +78,7 @@ TEST(PlatformViewShell, UpdateSemanticsDoesFlutterViewUpdateSemantics) { delegate->UpdateSemantics(update, actions); } -TEST(PlatformViewShell, UpdateSemanticsDoesUpdatelinkUrl) { +TEST(PlatformViewShell, UpdateSemanticsDoesUpdateLinkUrl) { auto jni_mock = std::make_shared(); auto delegate = std::make_unique(jni_mock); @@ -124,8 +125,76 @@ TEST(PlatformViewShell, UpdateSemanticsDoesUpdatelinkUrl) { buffer_int32[position++] = -1; // node0.hint buffer_int32[position++] = -1; // node0.hintAttributes buffer_int32[position++] = -1; // node0.tooltip - buffer_int32[position++] = expected_strings.size(); // node0.tooltip + buffer_int32[position++] = expected_strings.size(); // node0.linkUrl expected_strings.push_back(node0.linkUrl); + buffer_int32[position++] = -1; // node0.locale + buffer_int32[position++] = node0.textDirection; + buffer_float32[position++] = node0.rect.left(); + buffer_float32[position++] = node0.rect.top(); + buffer_float32[position++] = node0.rect.right(); + buffer_float32[position++] = node0.rect.bottom(); + node0.transform.getColMajor(&buffer_float32[position]); + position += 16; + buffer_int32[position++] = 0; // node0.childrenInTraversalOrder.size(); + buffer_int32[position++] = 0; // node0.customAccessibilityActions.size(); + EXPECT_CALL(*jni_mock, + FlutterViewUpdateSemantics(expected_buffer, expected_strings, + expected_string_attribute_args)); + // Creates empty custom actions. + flutter::CustomAccessibilityActionUpdates actions; + delegate->UpdateSemantics(update, actions); +} + +TEST(PlatformViewShell, UpdateSemanticsDoesUpdateLocale) { + auto jni_mock = std::make_shared(); + auto delegate = std::make_unique(jni_mock); + + flutter::SemanticsNodeUpdates update; + flutter::SemanticsNode node0; + node0.id = 0; + node0.identifier = "identifier"; + node0.label = "label"; + node0.locale = "es-MX"; + update.insert(std::make_pair(0, node0)); + + std::vector expected_buffer( + PlatformViewAndroidDelegate::kBytesPerNode); + std::vector> expected_string_attribute_args(0); + size_t position = 0; + int32_t* buffer_int32 = reinterpret_cast(&expected_buffer[0]); + float* buffer_float32 = reinterpret_cast(&expected_buffer[0]); + std::vector expected_strings; + buffer_int32[position++] = node0.id; + std::memcpy(&buffer_int32[position], &node0.flags, 2); + position += 2; + buffer_int32[position++] = node0.actions; + buffer_int32[position++] = node0.maxValueLength; + buffer_int32[position++] = node0.currentValueLength; + buffer_int32[position++] = node0.textSelectionBase; + buffer_int32[position++] = node0.textSelectionExtent; + buffer_int32[position++] = node0.platformViewId; + buffer_int32[position++] = node0.scrollChildren; + buffer_int32[position++] = node0.scrollIndex; + buffer_float32[position++] = static_cast(node0.scrollPosition); + buffer_float32[position++] = static_cast(node0.scrollExtentMax); + buffer_float32[position++] = static_cast(node0.scrollExtentMin); + buffer_int32[position++] = expected_strings.size(); // node0.identifier + expected_strings.push_back(node0.identifier); + buffer_int32[position++] = expected_strings.size(); // node0.label + expected_strings.push_back(node0.label); + buffer_int32[position++] = -1; // node0.labelAttributes + buffer_int32[position++] = -1; // node0.value + buffer_int32[position++] = -1; // node0.valueAttributes + buffer_int32[position++] = -1; // node0.increasedValue + buffer_int32[position++] = -1; // node0.increasedValueAttributes + buffer_int32[position++] = -1; // node0.decreasedValue + buffer_int32[position++] = -1; // node0.decreasedValueAttributes + buffer_int32[position++] = -1; // node0.hint + buffer_int32[position++] = -1; // node0.hintAttributes + buffer_int32[position++] = -1; // node0.tooltip + buffer_int32[position++] = -1; // node0.linkUrl + buffer_int32[position++] = expected_strings.size(); + expected_strings.push_back(node0.locale); // node0.locale buffer_int32[position++] = node0.textDirection; buffer_float32[position++] = node0.rect.left(); buffer_float32[position++] = node0.rect.top(); @@ -219,6 +288,7 @@ TEST(PlatformViewShell, {locale_attribute->locale.begin(), locale_attribute->locale.end()}); buffer_int32[position++] = -1; // node0.tooltip buffer_int32[position++] = -1; // node0.linkUrl + buffer_int32[position++] = -1; // node0.locale buffer_int32[position++] = node0.textDirection; buffer_float32[position++] = node0.rect.left(); buffer_float32[position++] = node0.rect.top(); diff --git a/engine/src/flutter/shell/platform/android/test/io/flutter/view/AccessibilityBridgeTest.java b/engine/src/flutter/shell/platform/android/test/io/flutter/view/AccessibilityBridgeTest.java index 8d6cd00b5a5..5b7e09d476a 100644 --- a/engine/src/flutter/shell/platform/android/test/io/flutter/view/AccessibilityBridgeTest.java +++ b/engine/src/flutter/shell/platform/android/test/io/flutter/view/AccessibilityBridgeTest.java @@ -114,7 +114,8 @@ public class AccessibilityBridgeTest { Context context = mock(Context.class); when(mockRootView.getContext()).thenReturn(context); final int position = 88; - // The getBoundsInScreen() in createAccessibilityNodeInfo() needs View.getLocationOnScreen() + // The getBoundsInScreen() in createAccessibilityNodeInfo() needs + // View.getLocationOnScreen() doAnswer( invocation -> { int[] outLocation = (int[]) invocation.getArguments()[0]; @@ -152,12 +153,12 @@ public class AccessibilityBridgeTest { when(context.getPackageName()).thenReturn("test"); when(mockManager.isTouchExplorationEnabled()).thenReturn(false); setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ mockChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ mockChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); verify(mockChannel).setAccessibilityFeatures(ACCESSIBILITY_FEATURE_NO_ANNOUNCE); } @@ -173,12 +174,12 @@ public class AccessibilityBridgeTest { when(mockManager.isTouchExplorationEnabled()).thenReturn(false); AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ mockChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ mockChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); ArgumentCaptor listenerCaptor = ArgumentCaptor.forClass(AccessibilityManager.TouchExplorationStateChangeListener.class); verify(mockManager).addTouchExplorationStateChangeListener(listenerCaptor.capture()); @@ -710,12 +711,12 @@ public class AccessibilityBridgeTest { when(context.getPackageName()).thenReturn("test"); AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ mockChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ mockChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); ViewParent mockParent = mock(ViewParent.class); when(mockRootView.getParent()).thenReturn(mockParent); @@ -749,12 +750,12 @@ public class AccessibilityBridgeTest { when(context.getPackageName()).thenReturn("test"); AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ mockChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ mockChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); ViewParent mockParent = mock(ViewParent.class); when(mockRootView.getParent()).thenReturn(mockParent); @@ -788,12 +789,12 @@ public class AccessibilityBridgeTest { when(context.getPackageName()).thenReturn("test"); AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ mockChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ mockChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); ViewParent mockParent = mock(ViewParent.class); when(mockRootView.getParent()).thenReturn(mockParent); @@ -842,6 +843,47 @@ public class AccessibilityBridgeTest { assertEquals(actual.getSpanEnd(spellOutSpan), 9); } + @Test + @Config(minSdk = API_LEVELS.FLUTTER_MIN) + public void itBuildsAttributedStringWithLocale() { + AccessibilityChannel mockChannel = mock(AccessibilityChannel.class); + AccessibilityViewEmbedder mockViewEmbedder = mock(AccessibilityViewEmbedder.class); + AccessibilityManager mockManager = mock(AccessibilityManager.class); + View mockRootView = mock(View.class); + Context context = mock(Context.class); + when(mockRootView.getContext()).thenReturn(context); + when(context.getPackageName()).thenReturn("test"); + AccessibilityBridge accessibilityBridge = + setUpBridge( + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ mockChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); + + ViewParent mockParent = mock(ViewParent.class); + when(mockRootView.getParent()).thenReturn(mockParent); + when(mockManager.isEnabled()).thenReturn(true); + + TestSemanticsNode root = new TestSemanticsNode(); + root.id = 0; + root.label = "label"; + root.locale = "es-MX"; + + TestSemanticsUpdate testSemanticsUpdate = root.toUpdate(); + testSemanticsUpdate.sendUpdateToBridge(accessibilityBridge); + AccessibilityNodeInfo nodeInfo = accessibilityBridge.createAccessibilityNodeInfo(0); + SpannableString actual = (SpannableString) nodeInfo.getContentDescription(); + assertEquals(actual.toString(), "label"); + Object[] objectSpans = actual.getSpans(0, actual.length(), Object.class); + assertEquals(objectSpans.length, 1); + LocaleSpan localeSpan = (LocaleSpan) objectSpans[0]; + assertEquals(localeSpan.getLocale().toLanguageTag(), "es-MX"); + assertEquals(actual.getSpanStart(localeSpan), 0); + assertEquals(actual.getSpanEnd(localeSpan), actual.length()); + } + @Test @Config(minSdk = API_LEVELS.FLUTTER_MIN) public void itSetsTextCorrectly() { @@ -854,12 +896,12 @@ public class AccessibilityBridgeTest { when(context.getPackageName()).thenReturn("test"); AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ mockChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ mockChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); ViewParent mockParent = mock(ViewParent.class); when(mockRootView.getParent()).thenReturn(mockParent); @@ -914,12 +956,12 @@ public class AccessibilityBridgeTest { when(context.getPackageName()).thenReturn("test"); AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ mockChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ mockChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); ViewParent mockParent = mock(ViewParent.class); when(mockRootView.getParent()).thenReturn(mockParent); @@ -953,12 +995,12 @@ public class AccessibilityBridgeTest { when(context.getPackageName()).thenReturn("test"); AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ mockChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ mockChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); ViewParent mockParent = mock(ViewParent.class); when(mockRootView.getParent()).thenReturn(mockParent); @@ -998,12 +1040,12 @@ public class AccessibilityBridgeTest { when(context.getPackageName()).thenReturn("test"); AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ mockChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ mockChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); ViewParent mockParent = mock(ViewParent.class); when(mockRootView.getParent()).thenReturn(mockParent); @@ -1036,12 +1078,12 @@ public class AccessibilityBridgeTest { when(context.getPackageName()).thenReturn("test"); AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ mockChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ mockChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); ViewParent mockParent = mock(ViewParent.class); when(mockRootView.getParent()).thenReturn(mockParent); @@ -1072,12 +1114,12 @@ public class AccessibilityBridgeTest { when(context.getPackageName()).thenReturn("test"); AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ mockChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ mockChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); ViewParent mockParent = mock(ViewParent.class); when(mockRootView.getParent()).thenReturn(mockParent); @@ -1118,12 +1160,12 @@ public class AccessibilityBridgeTest { when(context.getPackageName()).thenReturn("test"); AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ accessibilityChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ accessibilityChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); ViewParent mockParent = mock(ViewParent.class); when(mockRootView.getParent()).thenReturn(mockParent); @@ -1171,12 +1213,12 @@ public class AccessibilityBridgeTest { when(context.getPackageName()).thenReturn("test"); AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ mockChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ mockChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); ViewParent mockParent = mock(ViewParent.class); when(mockRootView.getParent()).thenReturn(mockParent); @@ -1225,19 +1267,20 @@ public class AccessibilityBridgeTest { AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ mockChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ mockChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); verify(mockChannel) .setAccessibilityFeatures( ACCESSIBILITY_FEATURE_BOLD_TEXT | ACCESSIBILITY_FEATURE_NO_ANNOUNCE); reset(mockChannel); - // Now verify that clearing the BOLD_TEXT flag doesn't touch any of the other flags. + // Now verify that clearing the BOLD_TEXT flag doesn't touch any of the other + // flags. // Ensure the DISABLE_ANIMATION flag will be set Settings.Global.putFloat(null, "transition_animation_scale", 0.0f); // Ensure the BOLD_TEXT flag will be cleared @@ -1245,14 +1288,15 @@ public class AccessibilityBridgeTest { accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ mockChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ mockChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); - // setAccessibilityFeatures() will be called multiple times from AccessibilityBridge's + // setAccessibilityFeatures() will be called multiple times from + // AccessibilityBridge's // constructor, verify that the latest argument is correct ArgumentCaptor captor = ArgumentCaptor.forClass(Integer.class); verify(mockChannel, atLeastOnce()).setAccessibilityFeatures(captor.capture()); @@ -1278,12 +1322,12 @@ public class AccessibilityBridgeTest { when(context.getPackageName()).thenReturn("test"); AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ accessibilityChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ accessibilityChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); ViewParent mockParent = mock(ViewParent.class); when(mockRootView.getParent()).thenReturn(mockParent); @@ -1340,12 +1384,12 @@ public class AccessibilityBridgeTest { when(context.getPackageName()).thenReturn("test"); AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ accessibilityChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ accessibilityChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); ViewParent mockParent = mock(ViewParent.class); when(mockRootView.getParent()).thenReturn(mockParent); @@ -1405,12 +1449,12 @@ public class AccessibilityBridgeTest { when(context.getPackageName()).thenReturn("test"); AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ mockChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ mockChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); ViewParent mockParent = mock(ViewParent.class); when(mockRootView.getParent()).thenReturn(mockParent); @@ -1463,12 +1507,12 @@ public class AccessibilityBridgeTest { when(context.getPackageName()).thenReturn("test"); AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ mockChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ mockChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); ViewParent mockParent = mock(ViewParent.class); when(mockRootView.getParent()).thenReturn(mockParent); @@ -1515,12 +1559,12 @@ public class AccessibilityBridgeTest { when(context.getPackageName()).thenReturn("test"); AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ mockChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ mockChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); ViewParent mockParent = mock(ViewParent.class); when(mockRootView.getParent()).thenReturn(mockParent); @@ -1564,12 +1608,12 @@ public class AccessibilityBridgeTest { when(context.getPackageName()).thenReturn("test"); AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ mockChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ mockChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); ViewParent mockParent = mock(ViewParent.class); when(mockRootView.getParent()).thenReturn(mockParent); @@ -1622,12 +1666,12 @@ public class AccessibilityBridgeTest { when(context.getPackageName()).thenReturn("test"); AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ mockChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ mockChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); ViewParent mockParent = mock(ViewParent.class); when(mockRootView.getParent()).thenReturn(mockParent); @@ -1681,12 +1725,12 @@ public class AccessibilityBridgeTest { when(context.getPackageName()).thenReturn("test"); AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ mockRootView, - /*accessibilityChannel=*/ mockChannel, - /*accessibilityManager=*/ mockManager, - /*contentResolver=*/ null, - /*accessibilityViewEmbedder=*/ mockViewEmbedder, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ mockRootView, + /* accessibilityChannel= */ mockChannel, + /* accessibilityManager= */ mockManager, + /* contentResolver= */ null, + /* accessibilityViewEmbedder= */ mockViewEmbedder, + /* platformViewsAccessibilityDelegate= */ null); ViewParent mockParent = mock(ViewParent.class); when(mockRootView.getParent()).thenReturn(mockParent); @@ -1790,9 +1834,9 @@ public class AccessibilityBridgeTest { AccessibilityBridge accessibilityBridge = setUpBridge( rootAccessibilityView, - /*accessibilityChannel=*/ null, - /*accessibilityManager=*/ null, - /*contentResolver=*/ null, + /* accessibilityChannel= */ null, + /* accessibilityManager= */ null, + /* contentResolver= */ null, accessibilityViewEmbedder, accessibilityDelegate); @@ -1835,9 +1879,9 @@ public class AccessibilityBridgeTest { AccessibilityBridge accessibilityBridge = setUpBridge( rootAccessibilityView, - /*accessibilityChannel=*/ null, - /*accessibilityManager=*/ null, - /*contentResolver=*/ null, + /* accessibilityChannel= */ null, + /* accessibilityManager= */ null, + /* contentResolver= */ null, accessibilityViewEmbedder, accessibilityDelegate); @@ -1870,9 +1914,9 @@ public class AccessibilityBridgeTest { AccessibilityBridge accessibilityBridge = setUpBridge( rootAccessibilityView, - /*accessibilityChannel=*/ null, - /*accessibilityManager=*/ null, - /*contentResolver=*/ null, + /* accessibilityChannel= */ null, + /* accessibilityManager= */ null, + /* contentResolver= */ null, accessibilityViewEmbedder, accessibilityDelegate); @@ -1908,10 +1952,10 @@ public class AccessibilityBridgeTest { AccessibilityViewEmbedder accessibilityViewEmbedder = mock(AccessibilityViewEmbedder.class); AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ null, - /*accessibilityChannel=*/ null, - /*accessibilityManager=*/ null, - /*contentResolver=*/ null, + /* rootAccessibilityView= */ null, + /* accessibilityChannel= */ null, + /* accessibilityManager= */ null, + /* contentResolver= */ null, accessibilityViewEmbedder, accessibilityDelegate); @@ -1936,14 +1980,15 @@ public class AccessibilityBridgeTest { AccessibilityBridge accessibilityBridge = setUpBridge( - /*rootAccessibilityView=*/ null, - /*accessibilityChannel=*/ mockChannel, - /*accessibilityManager=*/ null, - /*contentResolver=*/ mockContentResolver, - /*accessibilityViewEmbedder=*/ null, - /*platformViewsAccessibilityDelegate=*/ null); + /* rootAccessibilityView= */ null, + /* accessibilityChannel= */ mockChannel, + /* accessibilityManager= */ null, + /* contentResolver= */ mockContentResolver, + /* accessibilityViewEmbedder= */ null, + /* platformViewsAccessibilityDelegate= */ null); - // Capture the observer registered for Settings.Global.TRANSITION_ANIMATION_SCALE + // Capture the observer registered for + // Settings.Global.TRANSITION_ANIMATION_SCALE ArgumentCaptor observerCaptor = ArgumentCaptor.forClass(ContentObserver.class); verify(mockContentResolver) .registerContentObserver( @@ -2203,6 +2248,7 @@ public class AccessibilityBridgeTest { List hintAttributes; String tooltip = null; String linkUrl = null; + String locale = null; int textDirection = 0; float left = 0.0f; float top = 0.0f; @@ -2272,6 +2318,12 @@ public class AccessibilityBridgeTest { strings.add(linkUrl); bytes.putInt(strings.size() - 1); } + if (locale == null) { + bytes.putInt(-1); + } else { + strings.add(locale); + bytes.putInt(strings.size() - 1); + } bytes.putInt(textDirection); bytes.putFloat(left); bytes.putFloat(top);