add initial test attempts

dev/devicelab/bin/tasks/android_engine_flags_debug_test.dart

@@ -0,0 +1,10 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/tasks/android_engine_flags_test.dart';
+
+Future<void> main() async {
+  await task(androidEngineFlagsTest('debug'));
+}

dev/devicelab/bin/tasks/android_engine_flags_release_test.dart

@@ -0,0 +1,10 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/tasks/android_engine_flags_test.dart';
+
+Future<void> main() async {
+  await task(androidEngineFlagsTest('release'));
+}

dev/devicelab/lib/framework/android_utils.dart

@@ -0,0 +1,48 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:path/path.dart' as path;
+import 'package:xml/xml.dart';
+
+void addMetadataToManifest(String testDirectory, List<(String, String)> keyPairs) {
+  final String manifestPath = path.join(
+    testDirectory,
+    'android',
+    'app',
+    'src',
+    'main',
+    'AndroidManifest.xml',
+  );
+  final File file = File(manifestPath);
+
+  if (!file.existsSync()) {
+    throw Exception('AndroidManifest.xml not found at $manifestPath');
+  }
+
+  final String xmlStr = file.readAsStringSync();
+  final XmlDocument xmlDoc = XmlDocument.parse(xmlStr);
+  final XmlElement applicationNode = xmlDoc.findAllElements('application').first;
+
+  // Check if the meta-data node already exists.
+  for (final (String key, String value) in keyPairs) {
+    final Iterable<XmlElement> existingMetaData = applicationNode
+        .findAllElements('meta-data')
+        .where((XmlElement node) => node.getAttribute('android:name') == key);
+
+    if (existingMetaData.isNotEmpty) {
+      final XmlElement existingEntry = existingMetaData.first;
+      existingEntry.setAttribute('android:value', value);
+    } else {
+      final XmlElement metaData = XmlElement(XmlName('meta-data'), <XmlAttribute>[
+        XmlAttribute(XmlName('android:name'), key),
+        XmlAttribute(XmlName('android:value'), value),
+      ]);
+      applicationNode.children.add(metaData);
+    }
+  }
+
+  file.writeAsStringSync(xmlDoc.toXmlString(pretty: true, indent: '    '));
+}

dev/devicelab/lib/tasks/android_engine_flags_test.dart

@@ -0,0 +1,311 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:path/path.dart' as path;
+
+import '../framework/android_utils.dart';
+import '../framework/framework.dart';
+import '../framework/task_result.dart';
+import '../framework/utils.dart';
+
+typedef TestFunction = Future<TaskResult> Function();
+
+// TODO(camsim99): set enable-dart-profiling to release ok. also go through all defaults or file an issue
+TaskFunction androidEngineFlagsTest(String buildMode) {
+  final isReleaseMode = buildMode == 'release';
+  final List<TaskFunction> tests = [
+    // _testInvalidFlag(buildMode),
+    if (isReleaseMode) _testIllegalFlagInReleaseMode(),
+    // _testCommandLineFlagPrecedence(buildMode), TODO(camsim99): change approach to use vm service port
+  ];
+
+  return () async {
+    final List<TaskResult> results = [];
+    for (final test in tests) {
+      final TaskResult result = await test();
+      results.add(result);
+      if (result.failed) {
+        return result;
+      }
+    }
+
+    return TaskResult.success(null);
+  };
+}
+
+TaskFunction _testInvalidFlag(String buildMode) {
+  return () async {
+    section('Create new Flutter Android app');
+    final Directory tempDir = Directory.systemTemp.createTempSync(
+      'android_flutter_shell_args_test.',
+    );
+    const projectName = 'androidfluttershellargstest';
+
+    try {
+      await inDirectory(tempDir, () async {
+        await flutter(
+          'create',
+          options: <String>['--platforms', 'android', '--org', 'io.flutter.devicelab', projectName],
+        );
+      });
+
+      section('Insert metadata with valid and invalid flags into AndroidManifest.xml');
+      final metadataKeyPairs = <(String, String)>[
+        (
+          'io.flutter.embedding.android.AOTSharedLibraryName',
+          'something/completely/and/totally/invalid.so',
+        ),
+        ('io.flutter.embedding.android.ImpellerLazyShaderInitialization', 'true'),
+      ];
+      addMetadataToManifest(path.join(tempDir.path, projectName), metadataKeyPairs);
+
+      section('Run Flutter Android app with modified manifest');
+      final foundInvalidAotLibraryLog = Completer<bool>();
+      late Process run;
+
+      await inDirectory(path.join(tempDir.path, projectName), () async {
+        run = await startFlutter('run', options: <String>['--$buildMode', '--verbose']);
+      });
+
+      final StreamSubscription<void> stdout = run.stdout
+          .transform<String>(utf8.decoder)
+          .transform<String>(const LineSplitter())
+          .listen((String line) {
+            print('CAMILLE :$line');
+            if (line.contains(
+              "Skipping unsafe AOT shared library name flag: something/completely/and/totally/invalid.so. Please ensure that the library is vetted and placed in your application's internal storage.",
+            )) {
+              foundInvalidAotLibraryLog.complete(true);
+            }
+          });
+
+      section('Check that warning log for invalid AOT shared library name is in STDOUT');
+      final Object result = await Future.any(<Future<Object>>[
+        foundInvalidAotLibraryLog.future,
+        run.exitCode,
+      ]);
+
+      if (result is int) {
+        throw Exception('flutter run failed, exitCode=$result');
+      }
+
+      section('Stop listening to STDOUT');
+      await stdout.cancel();
+      run.kill();
+
+      return TaskResult.success(null);
+    } on TaskResult catch (taskResult) {
+      return taskResult;
+    } catch (e, stackTrace) {
+      print('Task exception stack trace:\n$stackTrace');
+      return TaskResult.failure(e.toString());
+    } finally {
+      rmTree(tempDir);
+    }
+  };
+}
+
+TaskFunction _testIllegalFlagInReleaseMode() {
+  return () async {
+    section('Create new Flutter Android app');
+    final Directory tempDir = Directory.systemTemp.createTempSync(
+      'android_flutter_shell_args_test.',
+    );
+    const projectName = 'androidfluttershellargstest';
+
+    try {
+      await inDirectory(tempDir, () async {
+        await flutter(
+          'create',
+          options: <String>['--platforms', 'android', '--org', 'io.flutter.devicelab', projectName],
+        );
+      });
+
+      section('Insert metadata only allowed in release mode for testing into AndroidManifest.xml');
+      final metadataKeyPairs = <(String, String)>[
+        ('io.flutter.embedding.android.UseTestFonts', 'true'),
+      ];
+      addMetadataToManifest(path.join(tempDir.path, projectName), metadataKeyPairs);
+
+      section('Run Flutter Android app with modified manifest');
+      final foundUseTestFontsLog = Completer<bool>();
+      late Process run;
+
+      await inDirectory(path.join(tempDir.path, projectName), () async {
+        run = await startFlutter('run', options: <String>['--release', '--verbose']);
+      });
+
+      final StreamSubscription<void> stdout = run.stdout
+          .transform<String>(utf8.decoder)
+          .transform<String>(const LineSplitter())
+          .listen((String line) {
+            if (line.contains(
+              'Flag with metadata key io.flutter.embedding.android.UseTestFonts is not allowed in release builds and will be ignored if specified in the application manifest or via the command line.',
+            )) {
+              foundUseTestFontsLog.complete(true);
+            }
+          });
+
+      section('Check that warning log for disallowed UseTestFonts flag is in STDOUT');
+      final Object result = await Future.any(<Future<Object>>[
+        foundUseTestFontsLog.future,
+        run.exitCode,
+      ]);
+
+      if (result is int) {
+        throw Exception('flutter run failed, exitCode=$result');
+      }
+
+      section('Stop listening to STDOUT');
+      await stdout.cancel();
+      run.kill();
+
+      return TaskResult.success(null);
+    } on TaskResult catch (taskResult) {
+      return taskResult;
+    } catch (e, stackTrace) {
+      print('Task exception stack trace:\n$stackTrace');
+      return TaskResult.failure(e.toString());
+    } finally {
+      rmTree(tempDir);
+    }
+  };
+}
+
+TaskFunction _testCommandLineFlagPrecedence(String buildMode) {
+  return () async {
+    section('Create new Flutter Android app');
+    final Directory tempDir = Directory.systemTemp.createTempSync(
+      'android_flutter_shell_args_test.',
+    );
+    const projectName = 'androidfluttershellargstest';
+
+    try {
+      await inDirectory(tempDir, () async {
+        await flutter(
+          'create',
+          options: <String>['--platforms', 'android', '--org', 'io.flutter.devicelab', projectName],
+        );
+      });
+
+      section('Create two assets files with different content for testing');
+      const assetsFileName = 'my_asset.txt';
+
+      final manifestAssetDir = Directory(path.join(tempDir.path, projectName, 'manifest_asset'));
+      await manifestAssetDir.create();
+      final manifestAssetFile = File(path.join(manifestAssetDir.path, assetsFileName));
+
+      const manifestAssetFileContentStr = 'Content from manifest asset directory';
+      await manifestAssetFile.writeAsString(manifestAssetFileContentStr);
+
+      final commandLineAssetDir = Directory(
+        path.join(tempDir.path, projectName, 'command_line_asset'),
+      );
+      await commandLineAssetDir.create();
+      final commandLineAssetFile = File(path.join(commandLineAssetDir.path, assetsFileName));
+
+      const commandLineAssetFileContentStr = 'Content from command line asset directory';
+      await commandLineAssetFile.writeAsString(commandLineAssetFileContentStr);
+
+      section('Insert metadata for manifest asset file into the manifest');
+      final metadataKeyPairs = <(String, String)>[
+        ('io.flutter.embedding.android.FlutterAssetsDir', manifestAssetDir.path),
+      ];
+
+      addMetadataToManifest(path.join(tempDir.path, projectName), metadataKeyPairs);
+
+      section('Modify main.dart to load and print asset content');
+      final mainFile = File(path.join(tempDir.path, projectName, 'lib', 'main.dart'));
+      final String originalMainContent = await mainFile.readAsString();
+      final String newMainContent = originalMainContent.replaceFirst('void main() {', '''
+import 'dart:async';
+import 'dart:io';
+
+import 'package:flutter/services.dart';
+
+void main() async {
+  WidgetsFlutterBinding.ensureInitialized();
+  String assetContent = '';
+  try {
+    assetContent = await rootBundle.loadString($assetsFileName);
+  } catch (e) {
+    assetContent = 'Error loading asset: \$e';
+  }
+  print('Asset Content: \$assetContent');
+        ''');
+
+      await mainFile.writeAsString(newMainContent);
+
+      section('Run Flutter Android app with modified manifest and --flutter-assets-dir');
+      final assetsContentFoundCompleter = Completer<String>();
+      late Process run;
+
+      await inDirectory(path.join(tempDir.path, projectName), () async {
+        run = await startFlutter(
+          'run',
+          options: <String>[
+            '--$buildMode',
+            '--flutter-assets-dir=${commandLineAssetDir.path}',
+            '--verbose',
+          ],
+        );
+      });
+      run.stderr.forEach(print);
+      final StreamSubscription<void> stdoutSubscription = run.stdout
+          .transform<String>(utf8.decoder)
+          .transform<String>(const LineSplitter())
+          .listen((String line) {
+            print('CAMILLE: $line');
+            if (line.contains('Asset Content: ')) {
+              assetsContentFoundCompleter.complete(
+                line.substring(line.indexOf('Asset Content: ') + 'Asset Content: '.length),
+              );
+            }
+          });
+
+      section('Check that manifest asset content is in STDOUT');
+      final Future<String> assetsContentFoundFuture = assetsContentFoundCompleter.future;
+      final Object result = await Future.any(<Future<Object>>[
+        assetsContentFoundFuture,
+        run.exitCode,
+      ]);
+
+      if (result is int) {
+        throw Exception('flutter run failed, exitCode=$result');
+      }
+
+      final String printedAssetContent = await assetsContentFoundFuture;
+      late TaskResult taskResult;
+      if (printedAssetContent == manifestAssetFileContentStr) {
+        taskResult = TaskResult.success(null);
+      } else if (printedAssetContent == commandLineAssetFileContentStr) {
+        taskResult = TaskResult.failure(
+          'Asset content defined on the command line did not take precedence over the manifest defined asset as expected.',
+        );
+      } else {
+        taskResult = TaskResult.failure(
+          'Neither asset file defined on the command line or the manifest was loaded correctly.',
+        );
+      }
+
+      section('Kill the app');
+      await stdoutSubscription.cancel();
+      run.kill();
+      await run.exitCode;
+
+      return taskResult;
+    } on TaskResult catch (taskResult) {
+      return taskResult;
+    } catch (e, stackTrace) {
+      print('Task exception stack trace:\n$stackTrace');
+      return TaskResult.failure(e.toString());
+    } finally {
+      rmTree(tempDir);
+    }
+  };
+}

dev/devicelab/lib/tasks/perf_tests.dart

@@ -13,6 +13,7 @@ import 'package:meta/meta.dart';
 import 'package:path/path.dart' as path;
 import 'package:xml/xml.dart';
 
+import '../framework/android_utils.dart';
 import '../framework/devices.dart';
 import '../framework/framework.dart';
 import '../framework/host_agent.dart';
@@ -820,58 +821,18 @@ Future<void> _resetPlist(String testDirectory) async {
   await exec('git', <String>['checkout', file.path]);
 }
 
-void _addMetadataToManifest(String testDirectory, List<(String, String)> keyPairs) {
-  final String manifestPath = path.join(
-    testDirectory,
-    'android',
-    'app',
-    'src',
-    'main',
-    'AndroidManifest.xml',
-  );
-  final file = File(manifestPath);
-
-  if (!file.existsSync()) {
-    throw Exception('AndroidManifest.xml not found at $manifestPath');
-  }
-
-  final String xmlStr = file.readAsStringSync();
-  final xmlDoc = XmlDocument.parse(xmlStr);
-  final XmlElement applicationNode = xmlDoc.findAllElements('application').first;
-
-  // Check if the meta-data node already exists.
-  for (final (String key, String value) in keyPairs) {
-    final Iterable<XmlElement> existingMetaData = applicationNode
-        .findAllElements('meta-data')
-        .where((XmlElement node) => node.getAttribute('android:name') == key);
-
-    if (existingMetaData.isNotEmpty) {
-      final XmlElement existingEntry = existingMetaData.first;
-      existingEntry.setAttribute('android:value', value);
-    } else {
-      final metaData = XmlElement(XmlName('meta-data'), <XmlAttribute>[
-        XmlAttribute(XmlName('android:name'), key),
-        XmlAttribute(XmlName('android:value'), value),
-      ]);
-      applicationNode.children.add(metaData);
-    }
-  }
-
-  file.writeAsStringSync(xmlDoc.toXmlString(pretty: true, indent: '    '));
-}
-
 void _addSurfaceControlSupportToManifest(String testDirectory) {
   final keyPairs = <(String, String)>[
     ('io.flutter.embedding.android.EnableSurfaceControl', 'true'),
   ];
-  _addMetadataToManifest(testDirectory, keyPairs);
+  addMetadataToManifest(testDirectory, keyPairs);
 }
 
 void _addMergedPlatformThreadSupportToManifest(String testDirectory) {
   final keyPairs = <(String, String)>[
     ('io.flutter.embedding.android.EnableMergedPlatformUIThread', 'true'),
   ];
-  _addMetadataToManifest(testDirectory, keyPairs);
+  addMetadataToManifest(testDirectory, keyPairs);
 }
 
 /// Opens the file at testDirectory + 'android/app/src/main/AndroidManifest.xml'
@@ -882,7 +843,7 @@ void _addVulkanGPUTracingToManifest(String testDirectory) {
   final keyPairs = <(String, String)>[
     ('io.flutter.embedding.android.EnableVulkanGPUTracing', 'true'),
   ];
-  _addMetadataToManifest(testDirectory, keyPairs);
+  addMetadataToManifest(testDirectory, keyPairs);
 }
 
 /// Opens the file at testDirectory + 'android/app/src/main/AndroidManifest.xml'
@@ -893,7 +854,7 @@ void _addLazyShaderMode(String testDirectory) {
   final keyPairs = <(String, String)>[
     ('io.flutter.embedding.android.ImpellerLazyShaderInitialization', 'true'),
   ];
-  _addMetadataToManifest(testDirectory, keyPairs);
+  addMetadataToManifest(testDirectory, keyPairs);
 }
 
 /// Opens the file at testDirectory + 'android/app/src/main/AndroidManifest.xml'
@@ -909,7 +870,7 @@ void _addOpenGLESToManifest(String testDirectory) {
     ('io.flutter.embedding.android.ImpellerBackend', 'opengles'),
     ('io.flutter.embedding.android.EnableOpenGLGPUTracing', 'true'),
   ];
-  _addMetadataToManifest(testDirectory, keyPairs);
+  addMetadataToManifest(testDirectory, keyPairs);
 }
 
 Future<void> _resetManifest(String testDirectory) async {