From 8a3751db2f057bf7994aa3b27912df8a23f43fb2 Mon Sep 17 00:00:00 2001 From: Camille Simon Date: Tue, 10 Feb 2026 16:49:46 -0800 Subject: [PATCH 1/4] add initial test attempts --- .../android_engine_flags_debug_test.dart | 10 + .../android_engine_flags_release_test.dart | 10 + .../lib/framework/android_utils.dart | 48 +++ .../lib/tasks/android_engine_flags_test.dart | 311 ++++++++++++++++++ dev/devicelab/lib/tasks/perf_tests.dart | 51 +-- 5 files changed, 385 insertions(+), 45 deletions(-) create mode 100644 dev/devicelab/bin/tasks/android_engine_flags_debug_test.dart create mode 100644 dev/devicelab/bin/tasks/android_engine_flags_release_test.dart create mode 100644 dev/devicelab/lib/framework/android_utils.dart create mode 100644 dev/devicelab/lib/tasks/android_engine_flags_test.dart diff --git a/dev/devicelab/bin/tasks/android_engine_flags_debug_test.dart b/dev/devicelab/bin/tasks/android_engine_flags_debug_test.dart new file mode 100644 index 00000000000..925c1a7cd87 --- /dev/null +++ b/dev/devicelab/bin/tasks/android_engine_flags_debug_test.dart @@ -0,0 +1,10 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/android_engine_flags_test.dart'; + +Future main() async { + await task(androidEngineFlagsTest('debug')); +} diff --git a/dev/devicelab/bin/tasks/android_engine_flags_release_test.dart b/dev/devicelab/bin/tasks/android_engine_flags_release_test.dart new file mode 100644 index 00000000000..6b44782fa23 --- /dev/null +++ b/dev/devicelab/bin/tasks/android_engine_flags_release_test.dart @@ -0,0 +1,10 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/android_engine_flags_test.dart'; + +Future main() async { + await task(androidEngineFlagsTest('release')); +} diff --git a/dev/devicelab/lib/framework/android_utils.dart b/dev/devicelab/lib/framework/android_utils.dart new file mode 100644 index 00000000000..191d4125167 --- /dev/null +++ b/dev/devicelab/lib/framework/android_utils.dart @@ -0,0 +1,48 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +import 'package:path/path.dart' as path; +import 'package:xml/xml.dart'; + +void addMetadataToManifest(String testDirectory, List<(String, String)> keyPairs) { + final String manifestPath = path.join( + testDirectory, + 'android', + 'app', + 'src', + 'main', + 'AndroidManifest.xml', + ); + final File file = File(manifestPath); + + if (!file.existsSync()) { + throw Exception('AndroidManifest.xml not found at $manifestPath'); + } + + final String xmlStr = file.readAsStringSync(); + final XmlDocument xmlDoc = XmlDocument.parse(xmlStr); + final XmlElement applicationNode = xmlDoc.findAllElements('application').first; + + // Check if the meta-data node already exists. + for (final (String key, String value) in keyPairs) { + final Iterable existingMetaData = applicationNode + .findAllElements('meta-data') + .where((XmlElement node) => node.getAttribute('android:name') == key); + + if (existingMetaData.isNotEmpty) { + final XmlElement existingEntry = existingMetaData.first; + existingEntry.setAttribute('android:value', value); + } else { + final XmlElement metaData = XmlElement(XmlName('meta-data'), [ + XmlAttribute(XmlName('android:name'), key), + XmlAttribute(XmlName('android:value'), value), + ]); + applicationNode.children.add(metaData); + } + } + + file.writeAsStringSync(xmlDoc.toXmlString(pretty: true, indent: ' ')); +} diff --git a/dev/devicelab/lib/tasks/android_engine_flags_test.dart b/dev/devicelab/lib/tasks/android_engine_flags_test.dart new file mode 100644 index 00000000000..b1472b63eea --- /dev/null +++ b/dev/devicelab/lib/tasks/android_engine_flags_test.dart @@ -0,0 +1,311 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:path/path.dart' as path; + +import '../framework/android_utils.dart'; +import '../framework/framework.dart'; +import '../framework/task_result.dart'; +import '../framework/utils.dart'; + +typedef TestFunction = Future Function(); + +// TODO(camsim99): set enable-dart-profiling to release ok. also go through all defaults or file an issue +TaskFunction androidEngineFlagsTest(String buildMode) { + final isReleaseMode = buildMode == 'release'; + final List tests = [ + // _testInvalidFlag(buildMode), + if (isReleaseMode) _testIllegalFlagInReleaseMode(), + // _testCommandLineFlagPrecedence(buildMode), TODO(camsim99): change approach to use vm service port + ]; + + return () async { + final List results = []; + for (final test in tests) { + final TaskResult result = await test(); + results.add(result); + if (result.failed) { + return result; + } + } + + return TaskResult.success(null); + }; +} + +TaskFunction _testInvalidFlag(String buildMode) { + return () async { + section('Create new Flutter Android app'); + final Directory tempDir = Directory.systemTemp.createTempSync( + 'android_flutter_shell_args_test.', + ); + const projectName = 'androidfluttershellargstest'; + + try { + await inDirectory(tempDir, () async { + await flutter( + 'create', + options: ['--platforms', 'android', '--org', 'io.flutter.devicelab', projectName], + ); + }); + + section('Insert metadata with valid and invalid flags into AndroidManifest.xml'); + final metadataKeyPairs = <(String, String)>[ + ( + 'io.flutter.embedding.android.AOTSharedLibraryName', + 'something/completely/and/totally/invalid.so', + ), + ('io.flutter.embedding.android.ImpellerLazyShaderInitialization', 'true'), + ]; + addMetadataToManifest(path.join(tempDir.path, projectName), metadataKeyPairs); + + section('Run Flutter Android app with modified manifest'); + final foundInvalidAotLibraryLog = Completer(); + late Process run; + + await inDirectory(path.join(tempDir.path, projectName), () async { + run = await startFlutter('run', options: ['--$buildMode', '--verbose']); + }); + + final StreamSubscription stdout = run.stdout + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((String line) { + print('CAMILLE :$line'); + if (line.contains( + "Skipping unsafe AOT shared library name flag: something/completely/and/totally/invalid.so. Please ensure that the library is vetted and placed in your application's internal storage.", + )) { + foundInvalidAotLibraryLog.complete(true); + } + }); + + section('Check that warning log for invalid AOT shared library name is in STDOUT'); + final Object result = await Future.any(>[ + foundInvalidAotLibraryLog.future, + run.exitCode, + ]); + + if (result is int) { + throw Exception('flutter run failed, exitCode=$result'); + } + + section('Stop listening to STDOUT'); + await stdout.cancel(); + run.kill(); + + return TaskResult.success(null); + } on TaskResult catch (taskResult) { + return taskResult; + } catch (e, stackTrace) { + print('Task exception stack trace:\n$stackTrace'); + return TaskResult.failure(e.toString()); + } finally { + rmTree(tempDir); + } + }; +} + +TaskFunction _testIllegalFlagInReleaseMode() { + return () async { + section('Create new Flutter Android app'); + final Directory tempDir = Directory.systemTemp.createTempSync( + 'android_flutter_shell_args_test.', + ); + const projectName = 'androidfluttershellargstest'; + + try { + await inDirectory(tempDir, () async { + await flutter( + 'create', + options: ['--platforms', 'android', '--org', 'io.flutter.devicelab', projectName], + ); + }); + + section('Insert metadata only allowed in release mode for testing into AndroidManifest.xml'); + final metadataKeyPairs = <(String, String)>[ + ('io.flutter.embedding.android.UseTestFonts', 'true'), + ]; + addMetadataToManifest(path.join(tempDir.path, projectName), metadataKeyPairs); + + section('Run Flutter Android app with modified manifest'); + final foundUseTestFontsLog = Completer(); + late Process run; + + await inDirectory(path.join(tempDir.path, projectName), () async { + run = await startFlutter('run', options: ['--release', '--verbose']); + }); + + final StreamSubscription stdout = run.stdout + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((String line) { + if (line.contains( + 'Flag with metadata key io.flutter.embedding.android.UseTestFonts is not allowed in release builds and will be ignored if specified in the application manifest or via the command line.', + )) { + foundUseTestFontsLog.complete(true); + } + }); + + section('Check that warning log for disallowed UseTestFonts flag is in STDOUT'); + final Object result = await Future.any(>[ + foundUseTestFontsLog.future, + run.exitCode, + ]); + + if (result is int) { + throw Exception('flutter run failed, exitCode=$result'); + } + + section('Stop listening to STDOUT'); + await stdout.cancel(); + run.kill(); + + return TaskResult.success(null); + } on TaskResult catch (taskResult) { + return taskResult; + } catch (e, stackTrace) { + print('Task exception stack trace:\n$stackTrace'); + return TaskResult.failure(e.toString()); + } finally { + rmTree(tempDir); + } + }; +} + +TaskFunction _testCommandLineFlagPrecedence(String buildMode) { + return () async { + section('Create new Flutter Android app'); + final Directory tempDir = Directory.systemTemp.createTempSync( + 'android_flutter_shell_args_test.', + ); + const projectName = 'androidfluttershellargstest'; + + try { + await inDirectory(tempDir, () async { + await flutter( + 'create', + options: ['--platforms', 'android', '--org', 'io.flutter.devicelab', projectName], + ); + }); + + section('Create two assets files with different content for testing'); + const assetsFileName = 'my_asset.txt'; + + final manifestAssetDir = Directory(path.join(tempDir.path, projectName, 'manifest_asset')); + await manifestAssetDir.create(); + final manifestAssetFile = File(path.join(manifestAssetDir.path, assetsFileName)); + + const manifestAssetFileContentStr = 'Content from manifest asset directory'; + await manifestAssetFile.writeAsString(manifestAssetFileContentStr); + + final commandLineAssetDir = Directory( + path.join(tempDir.path, projectName, 'command_line_asset'), + ); + await commandLineAssetDir.create(); + final commandLineAssetFile = File(path.join(commandLineAssetDir.path, assetsFileName)); + + const commandLineAssetFileContentStr = 'Content from command line asset directory'; + await commandLineAssetFile.writeAsString(commandLineAssetFileContentStr); + + section('Insert metadata for manifest asset file into the manifest'); + final metadataKeyPairs = <(String, String)>[ + ('io.flutter.embedding.android.FlutterAssetsDir', manifestAssetDir.path), + ]; + + addMetadataToManifest(path.join(tempDir.path, projectName), metadataKeyPairs); + + section('Modify main.dart to load and print asset content'); + final mainFile = File(path.join(tempDir.path, projectName, 'lib', 'main.dart')); + final String originalMainContent = await mainFile.readAsString(); + final String newMainContent = originalMainContent.replaceFirst('void main() {', ''' +import 'dart:async'; +import 'dart:io'; + +import 'package:flutter/services.dart'; + +void main() async { + WidgetsFlutterBinding.ensureInitialized(); + String assetContent = ''; + try { + assetContent = await rootBundle.loadString($assetsFileName); + } catch (e) { + assetContent = 'Error loading asset: \$e'; + } + print('Asset Content: \$assetContent'); + '''); + + await mainFile.writeAsString(newMainContent); + + section('Run Flutter Android app with modified manifest and --flutter-assets-dir'); + final assetsContentFoundCompleter = Completer(); + late Process run; + + await inDirectory(path.join(tempDir.path, projectName), () async { + run = await startFlutter( + 'run', + options: [ + '--$buildMode', + '--flutter-assets-dir=${commandLineAssetDir.path}', + '--verbose', + ], + ); + }); + run.stderr.forEach(print); + final StreamSubscription stdoutSubscription = run.stdout + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((String line) { + print('CAMILLE: $line'); + if (line.contains('Asset Content: ')) { + assetsContentFoundCompleter.complete( + line.substring(line.indexOf('Asset Content: ') + 'Asset Content: '.length), + ); + } + }); + + section('Check that manifest asset content is in STDOUT'); + final Future assetsContentFoundFuture = assetsContentFoundCompleter.future; + final Object result = await Future.any(>[ + assetsContentFoundFuture, + run.exitCode, + ]); + + if (result is int) { + throw Exception('flutter run failed, exitCode=$result'); + } + + final String printedAssetContent = await assetsContentFoundFuture; + late TaskResult taskResult; + if (printedAssetContent == manifestAssetFileContentStr) { + taskResult = TaskResult.success(null); + } else if (printedAssetContent == commandLineAssetFileContentStr) { + taskResult = TaskResult.failure( + 'Asset content defined on the command line did not take precedence over the manifest defined asset as expected.', + ); + } else { + taskResult = TaskResult.failure( + 'Neither asset file defined on the command line or the manifest was loaded correctly.', + ); + } + + section('Kill the app'); + await stdoutSubscription.cancel(); + run.kill(); + await run.exitCode; + + return taskResult; + } on TaskResult catch (taskResult) { + return taskResult; + } catch (e, stackTrace) { + print('Task exception stack trace:\n$stackTrace'); + return TaskResult.failure(e.toString()); + } finally { + rmTree(tempDir); + } + }; +} diff --git a/dev/devicelab/lib/tasks/perf_tests.dart b/dev/devicelab/lib/tasks/perf_tests.dart index bed49b17deb..ae3a55dea48 100644 --- a/dev/devicelab/lib/tasks/perf_tests.dart +++ b/dev/devicelab/lib/tasks/perf_tests.dart @@ -13,6 +13,7 @@ import 'package:meta/meta.dart'; import 'package:path/path.dart' as path; import 'package:xml/xml.dart'; +import '../framework/android_utils.dart'; import '../framework/devices.dart'; import '../framework/framework.dart'; import '../framework/host_agent.dart'; @@ -820,58 +821,18 @@ Future _resetPlist(String testDirectory) async { await exec('git', ['checkout', file.path]); } -void _addMetadataToManifest(String testDirectory, List<(String, String)> keyPairs) { - final String manifestPath = path.join( - testDirectory, - 'android', - 'app', - 'src', - 'main', - 'AndroidManifest.xml', - ); - final file = File(manifestPath); - - if (!file.existsSync()) { - throw Exception('AndroidManifest.xml not found at $manifestPath'); - } - - final String xmlStr = file.readAsStringSync(); - final xmlDoc = XmlDocument.parse(xmlStr); - final XmlElement applicationNode = xmlDoc.findAllElements('application').first; - - // Check if the meta-data node already exists. - for (final (String key, String value) in keyPairs) { - final Iterable existingMetaData = applicationNode - .findAllElements('meta-data') - .where((XmlElement node) => node.getAttribute('android:name') == key); - - if (existingMetaData.isNotEmpty) { - final XmlElement existingEntry = existingMetaData.first; - existingEntry.setAttribute('android:value', value); - } else { - final metaData = XmlElement(XmlName('meta-data'), [ - XmlAttribute(XmlName('android:name'), key), - XmlAttribute(XmlName('android:value'), value), - ]); - applicationNode.children.add(metaData); - } - } - - file.writeAsStringSync(xmlDoc.toXmlString(pretty: true, indent: ' ')); -} - void _addSurfaceControlSupportToManifest(String testDirectory) { final keyPairs = <(String, String)>[ ('io.flutter.embedding.android.EnableSurfaceControl', 'true'), ]; - _addMetadataToManifest(testDirectory, keyPairs); + addMetadataToManifest(testDirectory, keyPairs); } void _addMergedPlatformThreadSupportToManifest(String testDirectory) { final keyPairs = <(String, String)>[ ('io.flutter.embedding.android.EnableMergedPlatformUIThread', 'true'), ]; - _addMetadataToManifest(testDirectory, keyPairs); + addMetadataToManifest(testDirectory, keyPairs); } /// Opens the file at testDirectory + 'android/app/src/main/AndroidManifest.xml' @@ -882,7 +843,7 @@ void _addVulkanGPUTracingToManifest(String testDirectory) { final keyPairs = <(String, String)>[ ('io.flutter.embedding.android.EnableVulkanGPUTracing', 'true'), ]; - _addMetadataToManifest(testDirectory, keyPairs); + addMetadataToManifest(testDirectory, keyPairs); } /// Opens the file at testDirectory + 'android/app/src/main/AndroidManifest.xml' @@ -893,7 +854,7 @@ void _addLazyShaderMode(String testDirectory) { final keyPairs = <(String, String)>[ ('io.flutter.embedding.android.ImpellerLazyShaderInitialization', 'true'), ]; - _addMetadataToManifest(testDirectory, keyPairs); + addMetadataToManifest(testDirectory, keyPairs); } /// Opens the file at testDirectory + 'android/app/src/main/AndroidManifest.xml' @@ -909,7 +870,7 @@ void _addOpenGLESToManifest(String testDirectory) { ('io.flutter.embedding.android.ImpellerBackend', 'opengles'), ('io.flutter.embedding.android.EnableOpenGLGPUTracing', 'true'), ]; - _addMetadataToManifest(testDirectory, keyPairs); + addMetadataToManifest(testDirectory, keyPairs); } Future _resetManifest(String testDirectory) async { From c56b862c4ce6acdd9f06c70a5df71b78d948529d Mon Sep 17 00:00:00 2001 From: Camille Simon Date: Wed, 11 Feb 2026 10:16:44 -0800 Subject: [PATCH 2/4] play with vm service port --- .../lib/framework/android_utils.dart | 6 +- .../lib/tasks/android_engine_flags_test.dart | 118 +++++++++++++++++- 2 files changed, 119 insertions(+), 5 deletions(-) diff --git a/dev/devicelab/lib/framework/android_utils.dart b/dev/devicelab/lib/framework/android_utils.dart index 191d4125167..ae6ddef80dd 100644 --- a/dev/devicelab/lib/framework/android_utils.dart +++ b/dev/devicelab/lib/framework/android_utils.dart @@ -16,14 +16,14 @@ void addMetadataToManifest(String testDirectory, List<(String, String)> keyPairs 'main', 'AndroidManifest.xml', ); - final File file = File(manifestPath); + final file = File(manifestPath); if (!file.existsSync()) { throw Exception('AndroidManifest.xml not found at $manifestPath'); } final String xmlStr = file.readAsStringSync(); - final XmlDocument xmlDoc = XmlDocument.parse(xmlStr); + final xmlDoc = XmlDocument.parse(xmlStr); final XmlElement applicationNode = xmlDoc.findAllElements('application').first; // Check if the meta-data node already exists. @@ -36,7 +36,7 @@ void addMetadataToManifest(String testDirectory, List<(String, String)> keyPairs final XmlElement existingEntry = existingMetaData.first; existingEntry.setAttribute('android:value', value); } else { - final XmlElement metaData = XmlElement(XmlName('meta-data'), [ + final metaData = XmlElement(XmlName('meta-data'), [ XmlAttribute(XmlName('android:name'), key), XmlAttribute(XmlName('android:value'), value), ]); diff --git a/dev/devicelab/lib/tasks/android_engine_flags_test.dart b/dev/devicelab/lib/tasks/android_engine_flags_test.dart index b1472b63eea..721487ec6ef 100644 --- a/dev/devicelab/lib/tasks/android_engine_flags_test.dart +++ b/dev/devicelab/lib/tasks/android_engine_flags_test.dart @@ -20,8 +20,8 @@ TaskFunction androidEngineFlagsTest(String buildMode) { final isReleaseMode = buildMode == 'release'; final List tests = [ // _testInvalidFlag(buildMode), - if (isReleaseMode) _testIllegalFlagInReleaseMode(), - // _testCommandLineFlagPrecedence(buildMode), TODO(camsim99): change approach to use vm service port + // if (isReleaseMode) _testIllegalFlagInReleaseMode(), + _testCommandLineFlagPrecedence(buildMode), ]; return () async { @@ -177,7 +177,121 @@ TaskFunction _testIllegalFlagInReleaseMode() { }; } +// TODO(camsim99): Refactor this into common location if I can. +Future getFreePort() async { + var port = 0; + final ServerSocket serverSocket = await ServerSocket.bind(InternetAddress.loopbackIPv4, 0); + port = serverSocket.port; + await serverSocket.close(); + return port; +} + TaskFunction _testCommandLineFlagPrecedence(String buildMode) { + return () async { + section('Create new Flutter Android app'); + final Directory tempDir = Directory.systemTemp.createTempSync( + 'android_flutter_shell_args_test.', + ); + const projectName = 'androidfluttershellargstest'; + + try { + await inDirectory(tempDir, () async { + await flutter( + 'create', + options: ['--platforms', 'android', '--org', 'io.flutter.devicelab', projectName], + ); + }); + + section('Retrieve two free ports for Dart VM service'); + final int manifestServicePort = await getFreePort(); + int commandLineServicePort = await getFreePort(); + while (commandLineServicePort == manifestServicePort) { + commandLineServicePort = await getFreePort(); + } + + section('Insert metadata for manifest VM service port into the manifest'); + final metadataKeyPairs = <(String, String)>[ + ('io.flutter.embedding.android.VMServicePort', manifestServicePort.toString()), + ]; + + addMetadataToManifest(path.join(tempDir.path, projectName), metadataKeyPairs); + + section('Run Flutter Android app with modified manifest and --vm-service-port='); + late Process run; + + await inDirectory(path.join(tempDir.path, projectName), () async { + run = await startFlutter( + 'run', + options: [ + '--$buildMode', + '--vm-service-port=$commandLineServicePort', + '--verbose', + ], + ); + }); + + section('Attempt to connect to VM service until available or timeout'); + final Duration timeout = const Duration(seconds: 30); + final Stopwatch stopwatch = Stopwatch()..start(); + Socket? socket; + while (socket == null && stopwatch.elapsed < timeout) { + try { + socket = await Socket.connect('127.0.0.1', commandLineServicePort); + } catch (e) { + // Ignore connection errors and retry. + await Future.delayed(const Duration(milliseconds: 500)); + } + } + + section('Check that the VM service is running on the specified port'); + if (socket == null) { + try { + await Socket.connect('127.0.0.1', manifestServicePort).timeout( + const Duration(seconds: 5), + onTimeout: () { + throw Exception( + 'Failed to connect to VM service on port $commandLineServicePort specified on the command line and also did not connect to $manifestServicePort specified in the manifest.', + ); + }, + ); + } catch (e) { + throw Exception( + 'Failed to connect to VM service on port $commandLineServicePort specified on the command line. Connected to $manifestServicePort specified in the manifest instead.', + ); + } + + // throw Exception( + // 'Incorrectly connecte to VM service on port $manifestServicePort specified in the manifest instead of $commandLineServicePort specified on the command line.', + // ); + // }, + // ); + + // throw Exception( + // 'Failed to connect to VM service on port $commandLineServicePort specified on the command line and also did not connect to $manifestServicePort specified in the manifest.', + // ); + // }, + // ); + } else { + socket.destroy(); + } + + section('Kill the app'); + run.kill(); + await run.exitCode; + + return TaskResult.success(null); + } on TaskResult catch (taskResult) { + return taskResult; + } catch (e, stackTrace) { + print('Task exception stack trace:\n$stackTrace'); + return TaskResult.failure(e.toString()); + } finally { + rmTree(tempDir); + } + }; +} + +TaskFunction _testCommandLineFlagPrecedence2(String buildMode) { return () async { section('Create new Flutter Android app'); final Directory tempDir = Directory.systemTemp.createTempSync( From 5dc3fcf13e9d8d825437f9a75209570a76e88782 Mon Sep 17 00:00:00 2001 From: Camille Simon Date: Wed, 11 Feb 2026 12:46:56 -0800 Subject: [PATCH 3/4] aot testing --- .../lib/tasks/android_engine_flags_test.dart | 86 +++++++------------ 1 file changed, 29 insertions(+), 57 deletions(-) diff --git a/dev/devicelab/lib/tasks/android_engine_flags_test.dart b/dev/devicelab/lib/tasks/android_engine_flags_test.dart index 721487ec6ef..98be52f44d6 100644 --- a/dev/devicelab/lib/tasks/android_engine_flags_test.dart +++ b/dev/devicelab/lib/tasks/android_engine_flags_test.dart @@ -186,7 +186,7 @@ Future getFreePort() async { return port; } -TaskFunction _testCommandLineFlagPrecedence(String buildMode) { +TaskFunction _testCommandLineFlagPrecedence2(String buildMode) { return () async { section('Create new Flutter Android app'); final Directory tempDir = Directory.systemTemp.createTempSync( @@ -202,21 +202,17 @@ TaskFunction _testCommandLineFlagPrecedence(String buildMode) { ); }); - section('Retrieve two free ports for Dart VM service'); - final int manifestServicePort = await getFreePort(); - int commandLineServicePort = await getFreePort(); - while (commandLineServicePort == manifestServicePort) { - commandLineServicePort = await getFreePort(); - } - section('Insert metadata for manifest VM service port into the manifest'); + const invalidAotSharedLibraryPath = 'something/completely/and/totally/invalid.so'; final metadataKeyPairs = <(String, String)>[ - ('io.flutter.embedding.android.VMServicePort', manifestServicePort.toString()), + ('io.flutter.embedding.android.AOTSharedLibraryName', invalidAotSharedLibraryPath), ]; addMetadataToManifest(path.join(tempDir.path, projectName), metadataKeyPairs); section('Run Flutter Android app with modified manifest and --vm-service-port='); + const validAotSharedLibraryPath = 'data/data/$projectName/'; + final foundInvalidAotLibraryLog = Completer(); late Process run; await inDirectory(path.join(tempDir.path, projectName), () async { @@ -224,61 +220,37 @@ TaskFunction _testCommandLineFlagPrecedence(String buildMode) { 'run', options: [ '--$buildMode', - '--vm-service-port=$commandLineServicePort', + '--aot-shared-library-name=$invalidAotSharedLibraryPath', '--verbose', ], ); }); - section('Attempt to connect to VM service until available or timeout'); - final Duration timeout = const Duration(seconds: 30); - final Stopwatch stopwatch = Stopwatch()..start(); - Socket? socket; - while (socket == null && stopwatch.elapsed < timeout) { - try { - socket = await Socket.connect('127.0.0.1', commandLineServicePort); - } catch (e) { - // Ignore connection errors and retry. - await Future.delayed(const Duration(milliseconds: 500)); - } + final StreamSubscription stdout = run.stdout + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((String line) { + print('CAMILLE :$line'); + if (line.contains( + "Skipping unsafe AOT shared library name flag: something/completely/and/totally/invalid.so. Please ensure that the library is vetted and placed in your application's internal storage.", + )) { + foundInvalidAotLibraryLog.complete(true); + } + }); + + section('Check that warning log for invalid AOT shared library name is in STDOUT'); + final Object result = await Future.any(>[ + foundInvalidAotLibraryLog.future, + run.exitCode, + ]); + + if (result is int) { + throw Exception('flutter run failed, exitCode=$result'); } - section('Check that the VM service is running on the specified port'); - if (socket == null) { - try { - await Socket.connect('127.0.0.1', manifestServicePort).timeout( - const Duration(seconds: 5), - onTimeout: () { - throw Exception( - 'Failed to connect to VM service on port $commandLineServicePort specified on the command line and also did not connect to $manifestServicePort specified in the manifest.', - ); - }, - ); - } catch (e) { - throw Exception( - 'Failed to connect to VM service on port $commandLineServicePort specified on the command line. Connected to $manifestServicePort specified in the manifest instead.', - ); - } - - // throw Exception( - // 'Incorrectly connecte to VM service on port $manifestServicePort specified in the manifest instead of $commandLineServicePort specified on the command line.', - // ); - // }, - // ); - - // throw Exception( - // 'Failed to connect to VM service on port $commandLineServicePort specified on the command line and also did not connect to $manifestServicePort specified in the manifest.', - // ); - // }, - // ); - } else { - socket.destroy(); - } - - section('Kill the app'); + section('Stop listening to STDOUT'); + await stdout.cancel(); run.kill(); - await run.exitCode; - return TaskResult.success(null); } on TaskResult catch (taskResult) { return taskResult; @@ -291,7 +263,7 @@ TaskFunction _testCommandLineFlagPrecedence(String buildMode) { }; } -TaskFunction _testCommandLineFlagPrecedence2(String buildMode) { +TaskFunction _testCommandLineFlagPrecedence(String buildMode) { return () async { section('Create new Flutter Android app'); final Directory tempDir = Directory.systemTemp.createTempSync( From 6622209c3a4140c947beab5c654b6026724b4c5e Mon Sep 17 00:00:00 2001 From: Camille Simon Date: Wed, 18 Feb 2026 16:25:53 -0800 Subject: [PATCH 4/4] add test flags test --- .../lib/framework/android_utils.dart | 6 +- .../lib/tasks/android_engine_flags_test.dart | 204 +++--------------- 2 files changed, 35 insertions(+), 175 deletions(-) diff --git a/dev/devicelab/lib/framework/android_utils.dart b/dev/devicelab/lib/framework/android_utils.dart index ae6ddef80dd..e876bfd3044 100644 --- a/dev/devicelab/lib/framework/android_utils.dart +++ b/dev/devicelab/lib/framework/android_utils.dart @@ -7,7 +7,7 @@ import 'dart:io'; import 'package:path/path.dart' as path; import 'package:xml/xml.dart'; -void addMetadataToManifest(String testDirectory, List<(String, String)> keyPairs) { +void addMetadataToManifest(String testDirectory, List<(String, String?)> keyPairs) { final String manifestPath = path.join( testDirectory, 'android', @@ -27,7 +27,7 @@ void addMetadataToManifest(String testDirectory, List<(String, String)> keyPairs final XmlElement applicationNode = xmlDoc.findAllElements('application').first; // Check if the meta-data node already exists. - for (final (String key, String value) in keyPairs) { + for (final (String key, String? value) in keyPairs) { final Iterable existingMetaData = applicationNode .findAllElements('meta-data') .where((XmlElement node) => node.getAttribute('android:name') == key); @@ -38,7 +38,7 @@ void addMetadataToManifest(String testDirectory, List<(String, String)> keyPairs } else { final metaData = XmlElement(XmlName('meta-data'), [ XmlAttribute(XmlName('android:name'), key), - XmlAttribute(XmlName('android:value'), value), + if (value != null) XmlAttribute(XmlName('android:value'), value), ]); applicationNode.children.add(metaData); } diff --git a/dev/devicelab/lib/tasks/android_engine_flags_test.dart b/dev/devicelab/lib/tasks/android_engine_flags_test.dart index 98be52f44d6..6fb9815b31d 100644 --- a/dev/devicelab/lib/tasks/android_engine_flags_test.dart +++ b/dev/devicelab/lib/tasks/android_engine_flags_test.dart @@ -15,13 +15,12 @@ import '../framework/utils.dart'; typedef TestFunction = Future Function(); -// TODO(camsim99): set enable-dart-profiling to release ok. also go through all defaults or file an issue TaskFunction androidEngineFlagsTest(String buildMode) { final isReleaseMode = buildMode == 'release'; final List tests = [ - // _testInvalidFlag(buildMode), - // if (isReleaseMode) _testIllegalFlagInReleaseMode(), - _testCommandLineFlagPrecedence(buildMode), + _testInvalidFlag(buildMode), + if (isReleaseMode) _testIllegalFlagInReleaseMode(), + if (!isReleaseMode) _testCommandLineFlagPrecedence(), ]; return () async { @@ -177,16 +176,7 @@ TaskFunction _testIllegalFlagInReleaseMode() { }; } -// TODO(camsim99): Refactor this into common location if I can. -Future getFreePort() async { - var port = 0; - final ServerSocket serverSocket = await ServerSocket.bind(InternetAddress.loopbackIPv4, 0); - port = serverSocket.port; - await serverSocket.close(); - return port; -} - -TaskFunction _testCommandLineFlagPrecedence2(String buildMode) { +TaskFunction _testCommandLineFlagPrecedence() { return () async { section('Create new Flutter Android app'); final Directory tempDir = Directory.systemTemp.createTempSync( @@ -202,55 +192,58 @@ TaskFunction _testCommandLineFlagPrecedence2(String buildMode) { ); }); - section('Insert metadata for manifest VM service port into the manifest'); - const invalidAotSharedLibraryPath = 'something/completely/and/totally/invalid.so'; - final metadataKeyPairs = <(String, String)>[ - ('io.flutter.embedding.android.AOTSharedLibraryName', invalidAotSharedLibraryPath), - ]; + section('Insert metadata for test flag into the manifest'); + final metadataKeyPairs = <(String, String?)>[('io.flutter.embedding.android.TestFlag', null)]; addMetadataToManifest(path.join(tempDir.path, projectName), metadataKeyPairs); - section('Run Flutter Android app with modified manifest and --vm-service-port='); - const validAotSharedLibraryPath = 'data/data/$projectName/'; - final foundInvalidAotLibraryLog = Completer(); + section('Run Flutter Android app with modified manifest and --test-flag'); + final metadataLineFoundFirst = Completer(); late Process run; await inDirectory(path.join(tempDir.path, projectName), () async { - run = await startFlutter( - 'run', - options: [ - '--$buildMode', - '--aot-shared-library-name=$invalidAotSharedLibraryPath', - '--verbose', - ], - ); + run = await startFlutter('run', options: ['--test-flag', '--verbose']); }); - final StreamSubscription stdout = run.stdout + final StreamSubscription stdoutSubscription = run.stdout .transform(utf8.decoder) .transform(const LineSplitter()) .listen((String line) { - print('CAMILLE :$line'); - if (line.contains( - "Skipping unsafe AOT shared library name flag: something/completely/and/totally/invalid.so. Please ensure that the library is vetted and placed in your application's internal storage.", + if (metadataLineFoundFirst.isCompleted) { + return; + } else if (line.contains( + 'For testing purposes only: test flag specified on the command line was loaded by the FlutterLoader.', )) { - foundInvalidAotLibraryLog.complete(true); + metadataLineFoundFirst.complete(false); + } else if (line.contains( + 'For testing purposes only: test flag specified in the manifest was loaded by the FlutterLoader.', + )) { + metadataLineFoundFirst.complete(true); } }); - section('Check that warning log for invalid AOT shared library name is in STDOUT'); + section('Check that the test flag logs are found in the expected order in STDOUT'); + final Future assetsContentFoundFuture = metadataLineFoundFirst.future; final Object result = await Future.any(>[ - foundInvalidAotLibraryLog.future, + assetsContentFoundFuture, run.exitCode, ]); if (result is int) { throw Exception('flutter run failed, exitCode=$result'); + } else if (result is bool) { + if (!result) { + throw Exception( + 'Test flags specified in the manifest unexpectedly took precedence over that specified on the command line.', + ); + } } - section('Stop listening to STDOUT'); - await stdout.cancel(); + section('Kill the app'); + await stdoutSubscription.cancel(); run.kill(); + await run.exitCode; + return TaskResult.success(null); } on TaskResult catch (taskResult) { return taskResult; @@ -262,136 +255,3 @@ TaskFunction _testCommandLineFlagPrecedence2(String buildMode) { } }; } - -TaskFunction _testCommandLineFlagPrecedence(String buildMode) { - return () async { - section('Create new Flutter Android app'); - final Directory tempDir = Directory.systemTemp.createTempSync( - 'android_flutter_shell_args_test.', - ); - const projectName = 'androidfluttershellargstest'; - - try { - await inDirectory(tempDir, () async { - await flutter( - 'create', - options: ['--platforms', 'android', '--org', 'io.flutter.devicelab', projectName], - ); - }); - - section('Create two assets files with different content for testing'); - const assetsFileName = 'my_asset.txt'; - - final manifestAssetDir = Directory(path.join(tempDir.path, projectName, 'manifest_asset')); - await manifestAssetDir.create(); - final manifestAssetFile = File(path.join(manifestAssetDir.path, assetsFileName)); - - const manifestAssetFileContentStr = 'Content from manifest asset directory'; - await manifestAssetFile.writeAsString(manifestAssetFileContentStr); - - final commandLineAssetDir = Directory( - path.join(tempDir.path, projectName, 'command_line_asset'), - ); - await commandLineAssetDir.create(); - final commandLineAssetFile = File(path.join(commandLineAssetDir.path, assetsFileName)); - - const commandLineAssetFileContentStr = 'Content from command line asset directory'; - await commandLineAssetFile.writeAsString(commandLineAssetFileContentStr); - - section('Insert metadata for manifest asset file into the manifest'); - final metadataKeyPairs = <(String, String)>[ - ('io.flutter.embedding.android.FlutterAssetsDir', manifestAssetDir.path), - ]; - - addMetadataToManifest(path.join(tempDir.path, projectName), metadataKeyPairs); - - section('Modify main.dart to load and print asset content'); - final mainFile = File(path.join(tempDir.path, projectName, 'lib', 'main.dart')); - final String originalMainContent = await mainFile.readAsString(); - final String newMainContent = originalMainContent.replaceFirst('void main() {', ''' -import 'dart:async'; -import 'dart:io'; - -import 'package:flutter/services.dart'; - -void main() async { - WidgetsFlutterBinding.ensureInitialized(); - String assetContent = ''; - try { - assetContent = await rootBundle.loadString($assetsFileName); - } catch (e) { - assetContent = 'Error loading asset: \$e'; - } - print('Asset Content: \$assetContent'); - '''); - - await mainFile.writeAsString(newMainContent); - - section('Run Flutter Android app with modified manifest and --flutter-assets-dir'); - final assetsContentFoundCompleter = Completer(); - late Process run; - - await inDirectory(path.join(tempDir.path, projectName), () async { - run = await startFlutter( - 'run', - options: [ - '--$buildMode', - '--flutter-assets-dir=${commandLineAssetDir.path}', - '--verbose', - ], - ); - }); - run.stderr.forEach(print); - final StreamSubscription stdoutSubscription = run.stdout - .transform(utf8.decoder) - .transform(const LineSplitter()) - .listen((String line) { - print('CAMILLE: $line'); - if (line.contains('Asset Content: ')) { - assetsContentFoundCompleter.complete( - line.substring(line.indexOf('Asset Content: ') + 'Asset Content: '.length), - ); - } - }); - - section('Check that manifest asset content is in STDOUT'); - final Future assetsContentFoundFuture = assetsContentFoundCompleter.future; - final Object result = await Future.any(>[ - assetsContentFoundFuture, - run.exitCode, - ]); - - if (result is int) { - throw Exception('flutter run failed, exitCode=$result'); - } - - final String printedAssetContent = await assetsContentFoundFuture; - late TaskResult taskResult; - if (printedAssetContent == manifestAssetFileContentStr) { - taskResult = TaskResult.success(null); - } else if (printedAssetContent == commandLineAssetFileContentStr) { - taskResult = TaskResult.failure( - 'Asset content defined on the command line did not take precedence over the manifest defined asset as expected.', - ); - } else { - taskResult = TaskResult.failure( - 'Neither asset file defined on the command line or the manifest was loaded correctly.', - ); - } - - section('Kill the app'); - await stdoutSubscription.cancel(); - run.kill(); - await run.exitCode; - - return taskResult; - } on TaskResult catch (taskResult) { - return taskResult; - } catch (e, stackTrace) { - print('Task exception stack trace:\n$stackTrace'); - return TaskResult.failure(e.toString()); - } finally { - rmTree(tempDir); - } - }; -}