diff --git a/dev/devicelab/bin/tasks/flutter_gallery_v2_chrome_run_test.dart b/dev/devicelab/bin/tasks/flutter_gallery_v2_chrome_run_test.dart new file mode 100644 index 00000000000..ad38fd39d87 --- /dev/null +++ b/dev/devicelab/bin/tasks/flutter_gallery_v2_chrome_run_test.dart @@ -0,0 +1,112 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/framework/utils.dart'; + +Future main() async { + await task(const NewGalleryChromeRunTest().run); +} + +/// URI for the New Flutter Gallery repository. +const String galleryRepo = 'https://github.com/flutter/gallery.git'; + +/// After the gallery loads, a duration of [durationToWaitForError] +/// is waited, allowing any possible exceptions to be thrown. +const Duration durationToWaitForError = Duration(seconds: 5); + +/// Flutter prints this string when an app is successfully loaded. +/// Used to check when the app is successfully loaded. +const String successfullyLoadedString = 'To hot restart'; + +/// Flutter prints this string when an exception is caught. +/// Used to check if there are any exceptions. +const String exceptionString = 'EXCEPTION CAUGHT'; + +/// Checks that the New Flutter Gallery runs successfully on Chrome. +class NewGalleryChromeRunTest { + const NewGalleryChromeRunTest(); + + /// Runs the test. + Future run() async { + await gitClone(path: 'temp', repo: galleryRepo); + + final TaskResult result = await inDirectory('temp/gallery', () async { + await flutter('doctor'); + await flutter('packages', options: ['get']); + + await flutter('build', options: [ + 'web', + '-v', + '--release', + '--no-pub', + ], environment: { + 'FLUTTER_WEB': 'true', + }); + + final List options = ['-d', 'chrome', '--verbose', '--resident']; + final Process process = await startProcess( + 'flutter', + flutterCommandArgs('run', options), + environment: { + 'FLUTTER_WEB': 'true', + }, + ); + + final Completer stdoutDone = Completer(); + final Completer stderrDone = Completer(); + + bool success = true; + + process.stdout + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((String line) { + if (line.contains(successfullyLoadedString)) { + // Successfully started. + Future.delayed( + durationToWaitForError, + () {process.stdin.write('q');} + ); + } + if (line.contains(exceptionString)) { + success = false; + } + print('stdout: $line'); + }, onDone: () { + stdoutDone.complete(); + }); + + process.stderr + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((String line) { + print('stderr: $line'); + }, onDone: () { + stderrDone.complete(); + }); + + await Future.wait(>[ + stdoutDone.future, + stderrDone.future, + ]); + + await process.exitCode; + + if (success) { + return TaskResult.success({}); + } else { + return TaskResult.failure('An exception was thrown.'); + } + }); + + rmTree(Directory('temp')); + + return result; + } +} diff --git a/dev/devicelab/bin/tasks/flutter_gallery_v2_web_compile_test.dart b/dev/devicelab/bin/tasks/flutter_gallery_v2_web_compile_test.dart new file mode 100644 index 00000000000..c3b24798829 --- /dev/null +++ b/dev/devicelab/bin/tasks/flutter_gallery_v2_web_compile_test.dart @@ -0,0 +1,44 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/framework/utils.dart'; + +import 'package:flutter_devicelab/tasks/perf_tests.dart' show WebCompileTest; + +Future main() async { + await task(const NewGalleryWebCompileTest().run); +} + +/// Measures the time to compile the New Flutter Gallery to JavaScript +/// and the size of the compiled code. +class NewGalleryWebCompileTest { + const NewGalleryWebCompileTest(); + + String get metricKeyPrefix => 'new_gallery'; + + /// Runs the test. + Future run() async { + await gitClone(path: 'temp', repo: 'https://github.com/flutter/gallery.git'); + + final Map metrics = await inDirectory>( + 'temp/gallery', + () async { + await flutter('doctor'); + + return await WebCompileTest.runSingleBuildTest( + directory: 'temp/gallery', + metric: metricKeyPrefix, + measureBuildTime: true, + ); + }, + ); + + rmTree(Directory('temp')); + + return TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList()); + } +} diff --git a/dev/devicelab/lib/framework/utils.dart b/dev/devicelab/lib/framework/utils.dart index f6fce13d2a6..d7dcb405a95 100644 --- a/dev/devicelab/lib/framework/utils.dart +++ b/dev/devicelab/lib/framework/utils.dart @@ -676,3 +676,18 @@ void checkFileContains(List patterns, String filePath) { } } } + +/// Clones a git repository. +/// +/// Removes the directory [path], then clones the git repository +/// specified by [repo] to the directory [path]. +Future gitClone({String path, String repo}) async { + rmTree(Directory(path)); + + await Directory(path).create(recursive: true); + + return await inDirectory( + path, + () => exec('git', ['clone', repo]), + ); +} diff --git a/dev/devicelab/lib/tasks/perf_tests.dart b/dev/devicelab/lib/tasks/perf_tests.dart index ac9c73f1c93..7b266fe274c 100644 --- a/dev/devicelab/lib/tasks/perf_tests.dart +++ b/dev/devicelab/lib/tasks/perf_tests.dart @@ -305,34 +305,17 @@ class WebCompileTest { Future run() async { final Map metrics = {}; - await inDirectory('${flutterDirectory.path}/examples/hello_world', () async { - await flutter('packages', options: ['get']); - await evalFlutter('build', options: [ - 'web', - '-v', - '--release', - '--no-pub', - ], environment: { - 'FLUTTER_WEB': 'true', - }); - final String output = '${flutterDirectory.path}/examples/hello_world/build/web/main.dart.js'; - await _measureSize('hello_world', output, metrics); - return null; - }); - await inDirectory('${flutterDirectory.path}/dev/integration_tests/flutter_gallery', () async { - await flutter('packages', options: ['get']); - await evalFlutter('build', options: [ - 'web', - '-v', - '--release', - '--no-pub', - ], environment: { - 'FLUTTER_WEB': 'true', - }); - final String output = '${flutterDirectory.path}/dev/integration_tests/flutter_gallery/build/web/main.dart.js'; - await _measureSize('flutter_gallery', output, metrics); - return null; - }); + + metrics.addAll(await runSingleBuildTest( + directory: '${flutterDirectory.path}/examples/hello_world', + metric: 'hello_world', + )); + + metrics.addAll(await runSingleBuildTest( + directory: '${flutterDirectory.path}/dev/integration_tests/flutter_gallery', + metric: 'flutter_gallery', + )); + const String sampleAppName = 'sample_flutter_app'; final Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName'); @@ -340,30 +323,61 @@ class WebCompileTest { await inDirectory(Directory.systemTemp, () async { await flutter('create', options: ['--template=app', sampleAppName], environment: { - 'FLUTTER_WEB': 'true', - }); - await inDirectory(sampleDir, () async { - await flutter('packages', options: ['get']); - await evalFlutter('build', options: [ - 'web', - '-v', - '--release', - '--no-pub', - ], environment: { - 'FLUTTER_WEB': 'true', - }); - await _measureSize('basic_material_app', path.join(sampleDir.path, 'build/web/main.dart.js'), metrics); + 'FLUTTER_WEB': 'true', }); }); + + metrics.addAll(await runSingleBuildTest( + directory: sampleDir.path, + metric: 'basic_material_app', + )); + return TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList()); } - static Future _measureSize(String metric, String output, Map metrics) async { - final ProcessResult result = await Process.run('du', ['-k', output]); - await Process.run('gzip',['-k', '9', output]); - final ProcessResult resultGzip = await Process.run('du', ['-k', output + '.gz']); - metrics['${metric}_dart2js_size'] = _parseDu(result.stdout as String); - metrics['${metric}_dart2js_size_gzip'] = _parseDu(resultGzip.stdout as String); + /// Run a single web compile test and return its metrics. + /// + /// Run a single web compile test for the app under [directory], and store + /// its metrics with prefix [metric]. + static Future> runSingleBuildTest({String directory, String metric, bool measureBuildTime = false}) { + return inDirectory>(directory, () async { + final Map metrics = {}; + + await flutter('packages', options: ['get']); + final Stopwatch watch = measureBuildTime ? Stopwatch() : null; + watch?.start(); + await evalFlutter('build', options: [ + 'web', + '-v', + '--release', + '--no-pub', + ], environment: { + 'FLUTTER_WEB': 'true', + }); + watch?.stop(); + final String outputFileName = path.join(directory, 'build/web/main.dart.js'); + metrics.addAll(await getSize(outputFileName, metric: metric)); + + if (measureBuildTime) { + metrics['${metric}_dart2js_millis'] = watch.elapsedMilliseconds; + } + + return metrics; + }); + } + + /// Obtains the size and gzipped size of a file given by [fileName]. + static Future> getSize(String fileName, {String metric}) async { + final Map sizeMetrics = {}; + + final ProcessResult result = await Process.run('du', ['-k', fileName]); + sizeMetrics['${metric}_dart2js_size'] = _parseDu(result.stdout as String); + + await Process.run('gzip',['-k', '9', fileName]); + final ProcessResult resultGzip = await Process.run('du', ['-k', fileName + '.gz']); + sizeMetrics['${metric}_dart2js_size_gzip'] = _parseDu(resultGzip.stdout as String); + + return sizeMetrics; } static int _parseDu(String source) { diff --git a/dev/devicelab/manifest.yaml b/dev/devicelab/manifest.yaml index 51d8728718f..c564b416b77 100644 --- a/dev/devicelab/manifest.yaml +++ b/dev/devicelab/manifest.yaml @@ -811,3 +811,18 @@ tasks: # stage: devicelab_ios # required_agent_capabilities: ["mac/ios", "ios/gl-render-image"] # flaky: true + + flutter_gallery_v2_chrome_run_test: + description: > + Checks that the New Flutter Gallery runs successfully on Chrome. + stage: devicelab + required_agent_capabilities: ["linux/android"] + flaky: true + + flutter_gallery_v2_web_compile_test: + description: > + Measures the time to compile the New Flutter Gallery to JavaScript + and the size of the compiled code. + stage: devicelab + required_agent_capabilities: ["linux/android"] + flaky: true