Devicelab tests (Chrome run, Web compile) for New Flutter Gallery (#53096)

dev/devicelab/bin/tasks/flutter_gallery_v2_chrome_run_test.dart

@@ -0,0 +1,112 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
+
+Future<void> main() async {
+  await task(const NewGalleryChromeRunTest().run);
+}
+
+/// URI for the New Flutter Gallery repository.
+const String galleryRepo = 'https://github.com/flutter/gallery.git';
+
+/// After the gallery loads, a duration of [durationToWaitForError]
+/// is waited, allowing any possible exceptions to be thrown.
+const Duration durationToWaitForError = Duration(seconds: 5);
+
+/// Flutter prints this string when an app is successfully loaded.
+/// Used to check when the app is successfully loaded.
+const String successfullyLoadedString = 'To hot restart';
+
+/// Flutter prints this string when an exception is caught.
+/// Used to check if there are any exceptions.
+const String exceptionString = 'EXCEPTION CAUGHT';
+
+/// Checks that the New Flutter Gallery runs successfully on Chrome.
+class NewGalleryChromeRunTest {
+  const NewGalleryChromeRunTest();
+
+  /// Runs the test.
+  Future<TaskResult> run() async {
+    await gitClone(path: 'temp', repo: galleryRepo);
+
+    final TaskResult result = await inDirectory<TaskResult>('temp/gallery', () async {
+      await flutter('doctor');
+      await flutter('packages', options: <String>['get']);
+
+      await flutter('build', options: <String>[
+        'web',
+        '-v',
+        '--release',
+        '--no-pub',
+      ], environment: <String, String>{
+        'FLUTTER_WEB': 'true',
+      });
+
+      final List<String> options = <String>['-d', 'chrome', '--verbose', '--resident'];
+      final Process process = await startProcess(
+        'flutter',
+        flutterCommandArgs('run', options),
+        environment: <String, String>{
+          'FLUTTER_WEB': 'true',
+        },
+      );
+
+      final Completer<void> stdoutDone = Completer<void>();
+      final Completer<void> stderrDone = Completer<void>();
+
+      bool success = true;
+
+      process.stdout
+          .transform<String>(utf8.decoder)
+          .transform<String>(const LineSplitter())
+          .listen((String line) {
+        if (line.contains(successfullyLoadedString)) {
+          // Successfully started.
+          Future<void>.delayed(
+            durationToWaitForError,
+            () {process.stdin.write('q');}
+          );
+        }
+        if (line.contains(exceptionString)) {
+          success = false;
+        }
+        print('stdout: $line');
+      }, onDone: () {
+        stdoutDone.complete();
+      });
+
+      process.stderr
+          .transform<String>(utf8.decoder)
+          .transform<String>(const LineSplitter())
+          .listen((String line) {
+        print('stderr: $line');
+      }, onDone: () {
+        stderrDone.complete();
+      });
+
+      await Future.wait<void>(<Future<void>>[
+        stdoutDone.future,
+        stderrDone.future,
+      ]);
+
+      await process.exitCode;
+
+      if (success) {
+        return TaskResult.success(<String, dynamic>{});
+      } else {
+        return TaskResult.failure('An exception was thrown.');
+      }
+    });
+
+    rmTree(Directory('temp'));
+
+    return result;
+  }
+}

dev/devicelab/bin/tasks/flutter_gallery_v2_web_compile_test.dart

@@ -0,0 +1,44 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
+
+import 'package:flutter_devicelab/tasks/perf_tests.dart' show WebCompileTest;
+
+Future<void> main() async {
+  await task(const NewGalleryWebCompileTest().run);
+}
+
+/// Measures the time to compile the New Flutter Gallery to JavaScript
+/// and the size of the compiled code.
+class NewGalleryWebCompileTest {
+  const NewGalleryWebCompileTest();
+
+  String get metricKeyPrefix => 'new_gallery';
+
+  /// Runs the test.
+  Future<TaskResult> run() async {
+    await gitClone(path: 'temp', repo: 'https://github.com/flutter/gallery.git');
+
+    final Map<String, Object> metrics = await inDirectory<Map<String, int>>(
+      'temp/gallery',
+      () async {
+        await flutter('doctor');
+
+        return await WebCompileTest.runSingleBuildTest(
+          directory: 'temp/gallery',
+          metric: metricKeyPrefix,
+          measureBuildTime: true,
+        );
+      },
+    );
+
+    rmTree(Directory('temp'));
+
+    return TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList());
+  }
+}

dev/devicelab/lib/framework/utils.dart

@@ -676,3 +676,18 @@ void checkFileContains(List<Pattern> patterns, String filePath) {
     }
   }
 }
+
+/// Clones a git repository.
+///
+/// Removes the directory [path], then clones the git repository
+/// specified by [repo] to the directory [path].
+Future<int> gitClone({String path, String repo}) async {
+  rmTree(Directory(path));
+
+  await Directory(path).create(recursive: true);
+
+  return await inDirectory<int>(
+    path,
+        () => exec('git', <String>['clone', repo]),
+  );
+}

dev/devicelab/lib/tasks/perf_tests.dart

@@ -305,34 +305,17 @@ class WebCompileTest {
 
   Future<TaskResult> run() async {
     final Map<String, Object> metrics = <String, Object>{};
-    await inDirectory<TaskResult>('${flutterDirectory.path}/examples/hello_world', () async {
-      await flutter('packages', options: <String>['get']);
-      await evalFlutter('build', options: <String>[
-        'web',
-        '-v',
-        '--release',
-        '--no-pub',
-      ], environment: <String, String>{
-        'FLUTTER_WEB': 'true',
-      });
-      final String output = '${flutterDirectory.path}/examples/hello_world/build/web/main.dart.js';
-      await _measureSize('hello_world', output, metrics);
-      return null;
-    });
-    await inDirectory<TaskResult>('${flutterDirectory.path}/dev/integration_tests/flutter_gallery', () async {
-      await flutter('packages', options: <String>['get']);
-      await evalFlutter('build', options: <String>[
-        'web',
-        '-v',
-        '--release',
-        '--no-pub',
-      ], environment: <String, String>{
-        'FLUTTER_WEB': 'true',
-      });
-      final String output = '${flutterDirectory.path}/dev/integration_tests/flutter_gallery/build/web/main.dart.js';
-      await _measureSize('flutter_gallery', output, metrics);
-      return null;
-    });
+
+    metrics.addAll(await runSingleBuildTest(
+      directory: '${flutterDirectory.path}/examples/hello_world',
+      metric: 'hello_world',
+    ));
+
+    metrics.addAll(await runSingleBuildTest(
+      directory: '${flutterDirectory.path}/dev/integration_tests/flutter_gallery',
+      metric: 'flutter_gallery',
+    ));
+
     const String sampleAppName = 'sample_flutter_app';
     final Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName');
 
@@ -340,30 +323,61 @@ class WebCompileTest {
 
     await inDirectory<void>(Directory.systemTemp, () async {
       await flutter('create', options: <String>['--template=app', sampleAppName], environment: <String, String>{
-          'FLUTTER_WEB': 'true',
-        });
-      await inDirectory(sampleDir, () async {
-        await flutter('packages', options: <String>['get']);
-        await evalFlutter('build', options: <String>[
-          'web',
-          '-v',
-          '--release',
-          '--no-pub',
-        ], environment: <String, String>{
-          'FLUTTER_WEB': 'true',
-        });
-        await _measureSize('basic_material_app', path.join(sampleDir.path, 'build/web/main.dart.js'), metrics);
+        'FLUTTER_WEB': 'true',
       });
     });
+
+    metrics.addAll(await runSingleBuildTest(
+      directory: sampleDir.path,
+      metric: 'basic_material_app',
+    ));
+
     return TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList());
   }
 
-  static Future<void> _measureSize(String metric, String output, Map<String, Object> metrics) async {
-    final ProcessResult result = await Process.run('du', <String>['-k', output]);
-    await Process.run('gzip',<String>['-k', '9', output]);
-    final ProcessResult resultGzip = await Process.run('du', <String>['-k', output + '.gz']);
-    metrics['${metric}_dart2js_size'] = _parseDu(result.stdout as String);
-    metrics['${metric}_dart2js_size_gzip'] = _parseDu(resultGzip.stdout as String);
+  /// Run a single web compile test and return its metrics.
+  ///
+  /// Run a single web compile test for the app under [directory], and store
+  /// its metrics with prefix [metric].
+  static Future<Map<String, int>> runSingleBuildTest({String directory, String metric, bool measureBuildTime = false}) {
+    return inDirectory<Map<String, int>>(directory, () async {
+      final Map<String, int> metrics = <String, int>{};
+
+      await flutter('packages', options: <String>['get']);
+      final Stopwatch watch = measureBuildTime ? Stopwatch() : null;
+      watch?.start();
+      await evalFlutter('build', options: <String>[
+        'web',
+        '-v',
+        '--release',
+        '--no-pub',
+      ], environment: <String, String>{
+        'FLUTTER_WEB': 'true',
+      });
+      watch?.stop();
+      final String outputFileName = path.join(directory, 'build/web/main.dart.js');
+      metrics.addAll(await getSize(outputFileName, metric: metric));
+
+      if (measureBuildTime) {
+        metrics['${metric}_dart2js_millis'] = watch.elapsedMilliseconds;
+      }
+
+      return metrics;
+    });
+  }
+
+  /// Obtains the size and gzipped size of a file given by [fileName].
+  static Future<Map<String, int>> getSize(String fileName, {String metric}) async {
+    final Map<String, int> sizeMetrics = <String, int>{};
+
+    final ProcessResult result = await Process.run('du', <String>['-k', fileName]);
+    sizeMetrics['${metric}_dart2js_size'] = _parseDu(result.stdout as String);
+
+    await Process.run('gzip',<String>['-k', '9', fileName]);
+    final ProcessResult resultGzip = await Process.run('du', <String>['-k', fileName + '.gz']);
+    sizeMetrics['${metric}_dart2js_size_gzip'] = _parseDu(resultGzip.stdout as String);
+
+    return sizeMetrics;
   }
 
   static int _parseDu(String source) {

dev/devicelab/manifest.yaml

@@ -811,3 +811,18 @@ tasks:
   #   stage: devicelab_ios
   #   required_agent_capabilities: ["mac/ios", "ios/gl-render-image"]
   #   flaky: true
+
+  flutter_gallery_v2_chrome_run_test:
+    description: >
+      Checks that the New Flutter Gallery runs successfully on Chrome.
+    stage: devicelab
+    required_agent_capabilities: ["linux/android"]
+    flaky: true
+
+  flutter_gallery_v2_web_compile_test:
+    description: >
+      Measures the time to compile the New Flutter Gallery to JavaScript
+      and the size of the compiled code.
+    stage: devicelab
+    required_agent_capabilities: ["linux/android"]
+    flaky: true