From 71e01bf54865a4fdc2cd7e79e0a580e0cf752111 Mon Sep 17 00:00:00 2001 From: Jonah Williams Date: Fri, 13 Jul 2018 13:47:31 -0700 Subject: [PATCH] Add assistiveTechnologyEnabled flag to window (flutter/engine#5740) --- engine/src/flutter/lib/ui/hooks.dart | 5 ++++ engine/src/flutter/lib/ui/window.dart | 24 +++++++++++++++++++ engine/src/flutter/lib/ui/window/window.cc | 10 ++++++++ engine/src/flutter/lib/ui/window/window.h | 1 + .../src/flutter/runtime/runtime_controller.cc | 13 +++++++++- .../src/flutter/runtime/runtime_controller.h | 3 +++ engine/src/flutter/shell/common/engine.cc | 4 ++++ engine/src/flutter/shell/common/engine.h | 2 ++ .../src/flutter/shell/common/platform_view.cc | 4 ++++ .../src/flutter/shell/common/platform_view.h | 5 ++++ engine/src/flutter/shell/common/shell.cc | 14 +++++++++++ engine/src/flutter/shell/common/shell.h | 4 ++++ .../android/io/flutter/view/FlutterView.java | 5 ++++ .../android/platform_view_android_jni.cc | 12 ++++++++++ .../framework/Source/FlutterViewController.mm | 12 ++++++---- .../platform/darwin/ios/platform_view_ios.h | 6 +++++ .../platform/darwin/ios/platform_view_ios.mm | 11 +++++++++ 17 files changed, 129 insertions(+), 6 deletions(-) diff --git a/engine/src/flutter/lib/ui/hooks.dart b/engine/src/flutter/lib/ui/hooks.dart index 4f94a9ac962..d889a191738 100644 --- a/engine/src/flutter/lib/ui/hooks.dart +++ b/engine/src/flutter/lib/ui/hooks.dart @@ -70,6 +70,11 @@ void _updateSemanticsEnabled(bool enabled) { _invoke(window.onSemanticsEnabledChanged, window._onSemanticsEnabledChangedZone); } +void _updateAssistiveTechnologyEnabled(bool enabled) { + window._assistiveTechnologyEnabled = enabled; + _invoke(window._onAssistiveTechnologyEnabled, window._onAssistiveTechnologyEnabledZone); +} + void _dispatchPlatformMessage(String name, ByteData data, int responseId) { if (window.onPlatformMessage != null) { _invoke3( diff --git a/engine/src/flutter/lib/ui/window.dart b/engine/src/flutter/lib/ui/window.dart index f8dad72f154..26ca82f6e9c 100644 --- a/engine/src/flutter/lib/ui/window.dart +++ b/engine/src/flutter/lib/ui/window.dart @@ -637,6 +637,30 @@ class Window { bool get semanticsEnabled => _semanticsEnabled; bool _semanticsEnabled = false; + /// Whether the user is using assitive technologies to interact with the + /// application. + /// + /// This includes screen readers such as TalkBack on Android and VoiceOVer + /// on iOS, as well as hardware switches, and more. + /// + /// The [onAssistiveTechnologyEnabled] callback is called whenever this value + /// changes. + bool get assistiveTechnologyEnabled => _assistiveTechnologyEnabled; + bool _assistiveTechnologyEnabled = false; + + /// A callback that is invoked when the value of [assistiveTechnologyEnabled] + /// changes. + /// + /// The framework invokes this callback in the same zone in which the callback + /// was set. + VoidCallback get onAssistiveTechnologyEnabled => _onAssistiveTechnologyEnabled; + VoidCallback _onAssistiveTechnologyEnabled; + Zone _onAssistiveTechnologyEnabledZone; + set onAssistiveTechnologyEnabled(VoidCallback callback) { + _onAssistiveTechnologyEnabled = callback; + _onAssistiveTechnologyEnabledZone = Zone.current; + } + /// A callback that is invoked when the value of [semanticsEnabled] changes. /// /// The framework invokes this callback in the same zone in which the diff --git a/engine/src/flutter/lib/ui/window/window.cc b/engine/src/flutter/lib/ui/window/window.cc index 5878dd26e4b..e5bbdd73e19 100644 --- a/engine/src/flutter/lib/ui/window/window.cc +++ b/engine/src/flutter/lib/ui/window/window.cc @@ -199,6 +199,16 @@ void Window::UpdateSemanticsEnabled(bool enabled) { {ToDart(enabled)}); } +void Window::UpdateAssistiveTechnologyEnabled(bool enabled) { + tonic::DartState* dart_state = library_.dart_state().get(); + if (!dart_state) + return; + tonic::DartState::Scope scope(dart_state); + + DartInvokeField(library_.value(), "_updateAssistiveTechnologyEnabled", + {ToDart(enabled)}); +} + void Window::DispatchPlatformMessage(fxl::RefPtr message) { tonic::DartState* dart_state = library_.dart_state().get(); if (!dart_state) diff --git a/engine/src/flutter/lib/ui/window/window.h b/engine/src/flutter/lib/ui/window/window.h index 892812a363a..e10fb8503a6 100644 --- a/engine/src/flutter/lib/ui/window/window.h +++ b/engine/src/flutter/lib/ui/window/window.h @@ -54,6 +54,7 @@ class Window final { const std::string& country_code); void UpdateUserSettingsData(const std::string& data); void UpdateSemanticsEnabled(bool enabled); + void UpdateAssistiveTechnologyEnabled(bool enabled); void DispatchPlatformMessage(fxl::RefPtr message); void DispatchPointerDataPacket(const PointerDataPacket& packet); void DispatchSemanticsAction(int32_t id, diff --git a/engine/src/flutter/runtime/runtime_controller.cc b/engine/src/flutter/runtime/runtime_controller.cc index d4c191b4a5e..982d4d3cd61 100644 --- a/engine/src/flutter/runtime/runtime_controller.cc +++ b/engine/src/flutter/runtime/runtime_controller.cc @@ -122,7 +122,8 @@ std::unique_ptr RuntimeController::Clone() const { bool RuntimeController::FlushRuntimeStateToIsolate() { return SetViewportMetrics(window_data_.viewport_metrics) && SetLocale(window_data_.language_code, window_data_.country_code) && - SetSemanticsEnabled(window_data_.semantics_enabled); + SetSemanticsEnabled(window_data_.semantics_enabled) && + SetAssistiveTechnologyEnabled(window_data_.assistive_technology_enabled); } bool RuntimeController::SetViewportMetrics(const ViewportMetrics& metrics) { @@ -170,6 +171,16 @@ bool RuntimeController::SetSemanticsEnabled(bool enabled) { return false; } +bool RuntimeController::SetAssistiveTechnologyEnabled(bool enabled) { + window_data_.assistive_technology_enabled = enabled; + if (auto window = GetWindowIfAvailable()) { + window->UpdateAssistiveTechnologyEnabled(window_data_.assistive_technology_enabled); + return true; + } + + return false; +} + bool RuntimeController::BeginFrame(fxl::TimePoint frame_time) { if (auto window = GetWindowIfAvailable()) { window->BeginFrame(frame_time); diff --git a/engine/src/flutter/runtime/runtime_controller.h b/engine/src/flutter/runtime/runtime_controller.h index 07ec70a9121..ad9959091cd 100644 --- a/engine/src/flutter/runtime/runtime_controller.h +++ b/engine/src/flutter/runtime/runtime_controller.h @@ -47,6 +47,8 @@ class RuntimeController final : public WindowClient { bool SetSemanticsEnabled(bool enabled); + bool SetAssistiveTechnologyEnabled(bool enabled); + bool BeginFrame(fxl::TimePoint frame_time); bool NotifyIdle(int64_t deadline); @@ -80,6 +82,7 @@ class RuntimeController final : public WindowClient { std::string country_code; std::string user_settings_data = "{}"; bool semantics_enabled = false; + bool assistive_technology_enabled = false; }; RuntimeDelegate& client_; diff --git a/engine/src/flutter/shell/common/engine.cc b/engine/src/flutter/shell/common/engine.cc index 3f8896a224d..7765f594b44 100644 --- a/engine/src/flutter/shell/common/engine.cc +++ b/engine/src/flutter/shell/common/engine.cc @@ -339,6 +339,10 @@ void Engine::SetSemanticsEnabled(bool enabled) { runtime_controller_->SetSemanticsEnabled(enabled); } +void Engine::SetAssistiveTechnologyEnabled(bool enabled) { + runtime_controller_->SetAssistiveTechnologyEnabled(enabled); +} + void Engine::StopAnimator() { animator_->Stop(); } diff --git a/engine/src/flutter/shell/common/engine.h b/engine/src/flutter/shell/common/engine.h index eeacfc4b848..79b20755667 100644 --- a/engine/src/flutter/shell/common/engine.h +++ b/engine/src/flutter/shell/common/engine.h @@ -99,6 +99,8 @@ class Engine final : public blink::RuntimeDelegate { void SetSemanticsEnabled(bool enabled); + void SetAssistiveTechnologyEnabled(bool enabled); + void ScheduleFrame(bool regenerate_layer_tree = true) override; // |blink::RuntimeDelegate| diff --git a/engine/src/flutter/shell/common/platform_view.cc b/engine/src/flutter/shell/common/platform_view.cc index cb84fbcedd7..e1aaf5d858d 100644 --- a/engine/src/flutter/shell/common/platform_view.cc +++ b/engine/src/flutter/shell/common/platform_view.cc @@ -52,6 +52,10 @@ void PlatformView::SetSemanticsEnabled(bool enabled) { delegate_.OnPlatformViewSetSemanticsEnabled(*this, enabled); } +void PlatformView::SetAssistiveTechnologyEnabled(bool enabled) { + delegate_.OnPlatformViewSetAssistiveTechnologyEnabled(*this, enabled); +} + void PlatformView::SetViewportMetrics(const blink::ViewportMetrics& metrics) { delegate_.OnPlatformViewSetViewportMetrics(*this, metrics); } diff --git a/engine/src/flutter/shell/common/platform_view.h b/engine/src/flutter/shell/common/platform_view.h index 8273563f132..02e942fa449 100644 --- a/engine/src/flutter/shell/common/platform_view.h +++ b/engine/src/flutter/shell/common/platform_view.h @@ -58,6 +58,9 @@ class PlatformView { virtual void OnPlatformViewSetSemanticsEnabled(const PlatformView& view, bool enabled) = 0; + virtual void OnPlatformViewSetAssistiveTechnologyEnabled(const PlatformView& view, + bool enabled) = 0; + virtual void OnPlatformViewRegisterTexture( const PlatformView& view, std::shared_ptr texture) = 0; @@ -84,6 +87,8 @@ class PlatformView { virtual void SetSemanticsEnabled(bool enabled); + virtual void SetAssistiveTechnologyEnabled(bool enabled); + void SetViewportMetrics(const blink::ViewportMetrics& metrics); void NotifyCreated(); diff --git a/engine/src/flutter/shell/common/shell.cc b/engine/src/flutter/shell/common/shell.cc index e6cb6fc10a8..43c769142de 100644 --- a/engine/src/flutter/shell/common/shell.cc +++ b/engine/src/flutter/shell/common/shell.cc @@ -574,6 +574,20 @@ void Shell::OnPlatformViewSetSemanticsEnabled(const PlatformView& view, }); } +void Shell::OnPlatformViewSetAssistiveTechnologyEnabled(const PlatformView& view, + bool enabled) { + FXL_DCHECK(is_setup_); + FXL_DCHECK(&view == platform_view_.get()); + FXL_DCHECK(task_runners_.GetPlatformTaskRunner()->RunsTasksOnCurrentThread()); + + task_runners_.GetUITaskRunner()->PostTask( + [engine = engine_->GetWeakPtr(), enabled] { + if (engine) { + engine->SetAssistiveTechnologyEnabled(enabled); + } + }); +} + // |shell::PlatformView::Delegate| void Shell::OnPlatformViewRegisterTexture( const PlatformView& view, diff --git a/engine/src/flutter/shell/common/shell.h b/engine/src/flutter/shell/common/shell.h index 8f12bf0ae64..2aa98bb1d60 100644 --- a/engine/src/flutter/shell/common/shell.h +++ b/engine/src/flutter/shell/common/shell.h @@ -149,6 +149,10 @@ class Shell final : public PlatformView::Delegate, void OnPlatformViewSetSemanticsEnabled(const PlatformView& view, bool enabled) override; + // |shell::PlatformView::Delegate| + void OnPlatformViewSetAssistiveTechnologyEnabled(const PlatformView& view, + bool enabled) override; + // |shell::PlatformView::Delegate| void OnPlatformViewRegisterTexture( const PlatformView& view, diff --git a/engine/src/flutter/shell/platform/android/io/flutter/view/FlutterView.java b/engine/src/flutter/shell/platform/android/io/flutter/view/FlutterView.java index 8974a0bec5e..66e4bde3d5b 100644 --- a/engine/src/flutter/shell/platform/android/io/flutter/view/FlutterView.java +++ b/engine/src/flutter/shell/platform/android/io/flutter/view/FlutterView.java @@ -669,6 +669,9 @@ public class FlutterView extends SurfaceView private static native void nativeSetSemanticsEnabled(long nativePlatformViewAndroid, boolean enabled); + + private static native void nativeSetAssistiveTechnologyEnabled(long nativePlatformViewAndroid, + boolean enabled); private static native boolean nativeGetIsSoftwareRenderingEnabled(); @@ -812,11 +815,13 @@ public class FlutterView extends SurfaceView if (enabled) { mTouchExplorationEnabled = true; ensureAccessibilityEnabled(); + nativeSetAssistiveTechnologyEnabled(mNativeView.get(), true); } else { mTouchExplorationEnabled = false; if (mAccessibilityNodeProvider != null) { mAccessibilityNodeProvider.handleTouchExplorationExit(); } + nativeSetAssistiveTechnologyEnabled(mNativeView.get(), false); } resetWillNotDraw(); } diff --git a/engine/src/flutter/shell/platform/android/platform_view_android_jni.cc b/engine/src/flutter/shell/platform/android/platform_view_android_jni.cc index 0fe2af1c505..24c229581d4 100644 --- a/engine/src/flutter/shell/platform/android/platform_view_android_jni.cc +++ b/engine/src/flutter/shell/platform/android/platform_view_android_jni.cc @@ -429,6 +429,13 @@ static void SetSemanticsEnabled(JNIEnv* env, ANDROID_SHELL_HOLDER->GetPlatformView()->SetSemanticsEnabled(enabled); } +static void SetAssistiveTechnologyEnabled(JNIEnv* env, + jobject jcaller, + jlong shell_holder, + jboolean enabled) { + ANDROID_SHELL_HOLDER->GetPlatformView()->SetAssistiveTechnologyEnabled(enabled); +} + static jboolean GetIsSoftwareRendering(JNIEnv* env, jobject jcaller) { return FlutterMain::Get().GetSettings().enable_software_rendering; } @@ -600,6 +607,11 @@ bool PlatformViewAndroid::Register(JNIEnv* env) { .signature = "(JZ)V", .fnPtr = reinterpret_cast(&shell::SetSemanticsEnabled), }, + { + .name = "nativeSetAssistiveTechnologyEnabled", + .signature = "(JZ)V", + .fnPtr = reinterpret_cast(&shell::SetAssistiveTechnologyEnabled), + }, { .name = "nativeGetIsSoftwareRenderingEnabled", .signature = "()Z", diff --git a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm index 252d5e61e4b..d77c6bd3974 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm +++ b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm @@ -792,16 +792,18 @@ static inline blink::PointerData::DeviceKind DeviceKindFromTouchType(UITouch* to #pragma mark - Accessibility - (void)onAccessibilityStatusChanged:(NSNotification*)notification { + auto platformView = _shell->GetPlatformView(); #if TARGET_OS_SIMULATOR // There doesn't appear to be any way to determine whether the accessibility // inspector is enabled on the simulator. We conservatively always turn on the - // accessibility bridge in the simulator. - bool enabled = true; + // accessibility bridge in the simulator, but never assistive technology. + platformView->SetSemanticsEnabled(true); + platformView->SetAssistiveTechnologyEnabled(false); #else - bool enabled = UIAccessibilityIsVoiceOverRunning() || UIAccessibilityIsSwitchControlRunning() || - UIAccessibilityIsSpeakScreenEnabled(); + bool enabled = UIAccessibilityIsVoiceOverRunning() || UIAccessibilityIsSwitchControlRunning(); + platformView->SetSemanticsEnabled(enabled || UIAccessibilityIsSpeakScreenEnabled()); + platformView->SetAssistiveTechnologyEnabled(enabled); #endif - _shell->GetPlatformView()->SetSemanticsEnabled(enabled); } #pragma mark - Memory Notifications diff --git a/engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.h b/engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.h index 8d1de7a0c09..9bd758b55f8 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.h +++ b/engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.h @@ -57,6 +57,12 @@ class PlatformViewIOS final : public HeadlessPlatformViewIOS { // |shell::PlatformView| void SetSemanticsEnabled(bool enabled) override; + // |shell::PlatformView| + void SetAssistiveTechnologyEnabled(bool enabled) override; + + // |shell::PlatformView| + void HandlePlatformMessage( + fxl::RefPtr message) override; // |shell::PlatformView| void UpdateSemantics(blink::SemanticsNodeUpdates update, diff --git a/engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.mm b/engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.mm index f1cfc70e35f..df960dc2fd8 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.mm +++ b/engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.mm @@ -67,6 +67,17 @@ void PlatformViewIOS::SetSemanticsEnabled(bool enabled) { PlatformView::SetSemanticsEnabled(enabled); } +// |shell:PlatformView| +void PlatformViewIOS::SetAssistiveTechnologyEnabled(bool enabled) { + if (enabled && !accessibility_bridge_) { + accessibility_bridge_ = std::make_unique(owner_view_, this); + } + // Note: since the accessibility bridge is needed for both semantics and + // assistive technologies, but you cannot have the latter without the + // former, we only destroy the bridge in SetSemanticsEnabled and not here. + PlatformView::SetAssistiveTechnologyEnabled(enabled); +} + // |shell::PlatformView| void PlatformViewIOS::UpdateSemantics(blink::SemanticsNodeUpdates update, blink::CustomAccessibilityActionUpdates actions) {