From e778686e72e799e3555adf0777bee548eecb3103 Mon Sep 17 00:00:00 2001 From: Jonah Williams Date: Tue, 29 Oct 2019 13:23:27 -0700 Subject: [PATCH] Re-enable chrome dev mode tests (#43691) --- .../lib/tasks/web_dev_mode_tests.dart | 275 +++++++++--------- dev/devicelab/manifest.yaml | 38 ++- 2 files changed, 161 insertions(+), 152 deletions(-) diff --git a/dev/devicelab/lib/tasks/web_dev_mode_tests.dart b/dev/devicelab/lib/tasks/web_dev_mode_tests.dart index 78cf94eadea..6408b8586fc 100644 --- a/dev/devicelab/lib/tasks/web_dev_mode_tests.dart +++ b/dev/devicelab/lib/tasks/web_dev_mode_tests.dart @@ -9,155 +9,166 @@ import 'dart:io'; import 'package:path/path.dart' as path; import '../framework/framework.dart'; -import '../framework/running_processes.dart'; import '../framework/utils.dart'; final Directory _editedFlutterGalleryDir = dir(path.join(Directory.systemTemp.path, 'edited_flutter_gallery')); final Directory flutterGalleryDir = dir(path.join(flutterDirectory.path, 'examples/flutter_gallery')); +const String kInitialStartupTime = 'InitialStartupTime'; +const String kFirstRestartTime = 'FistRestartTime'; +const String kFirstRecompileTime = 'FirstRecompileTime'; +const String kSecondStartupTime = 'SecondStartupTime'; +const String kSecondRestartTime = 'SecondRestartTime'; + TaskFunction createWebDevModeTest() { return () async { final List options = [ - '--hot', '-d', 'chrome', '--verbose', '--resident', '--target=lib/main.dart', + '--hot', '-d', 'web-server', '--verbose', '--resident', '--target=lib/main.dart', ]; int hotRestartCount = 0; - String chromeProcessName; - if (Platform.isMacOS) { - chromeProcessName = 'Chrome'; - } else if (Platform.isLinux) { - chromeProcessName = 'chrome'; - } else if (Platform.isWindows) { - chromeProcessName = 'chrome.exe'; - } - final Set beforeChromeProcesses = await getRunningProcesses(processName: chromeProcessName) - .map((RunningProcessInfo info) => info.pid) - .toSet(); - try { - await inDirectory(flutterDirectory, () async { - rmTree(_editedFlutterGalleryDir); - mkdirs(_editedFlutterGalleryDir); - recursiveCopy(flutterGalleryDir, _editedFlutterGalleryDir); - await inDirectory(_editedFlutterGalleryDir, () async { - { - final Process packagesGet = await startProcess( - path.join(flutterDirectory.path, 'bin', 'flutter'), - ['packages', 'get'], - environment: { - 'FLUTTER_WEB': 'true', - }, - ); - await packagesGet.exitCode; - final Process process = await startProcess( - path.join(flutterDirectory.path, 'bin', 'flutter'), - flutterCommandArgs('run', options), - environment: { - 'FLUTTER_WEB': 'true', - }, - ); + final Map measurements = {}; + await inDirectory(flutterDirectory, () async { + rmTree(_editedFlutterGalleryDir); + mkdirs(_editedFlutterGalleryDir); + recursiveCopy(flutterGalleryDir, _editedFlutterGalleryDir); + await inDirectory(_editedFlutterGalleryDir, () async { + { + final Process packagesGet = await startProcess( + path.join(flutterDirectory.path, 'bin', 'flutter'), + ['packages', 'get'], + environment: { + 'FLUTTER_WEB': 'true', + }, + ); + await packagesGet.exitCode; + final Process process = await startProcess( + path.join(flutterDirectory.path, 'bin', 'flutter'), + flutterCommandArgs('run', options), + environment: { + 'FLUTTER_WEB': 'true', + }, + ); - final Completer stdoutDone = Completer(); - final Completer stderrDone = Completer(); - process.stdout - .transform(utf8.decoder) - .transform(const LineSplitter()) - .listen((String line) { - if (line.contains('To hot restart')) { - process.stdin.write('R'); - } - if (line.contains('Restarted')) { - if (hotRestartCount == 0) { - // Update the file and reload again. - final File appDartSource = file(path.join( - _editedFlutterGalleryDir.path, 'lib/gallery/app.dart', - )); - appDartSource.writeAsStringSync( - appDartSource.readAsStringSync().replaceFirst( - "'Flutter Gallery'", "'Updated Flutter Gallery'", - ) - ); - process.stdin.writeln('R'); - ++hotRestartCount; - } else { - // Quit after second hot restart. - process.stdin.writeln('q'); - } - } - print('stdout: $line'); - }, onDone: () { - stdoutDone.complete(); - }); - process.stderr - .transform(utf8.decoder) - .transform(const LineSplitter()) - .listen((String line) { - print('stderr: $line'); - }, onDone: () { - stderrDone.complete(); - }); - - await Future.wait(>[ - stdoutDone.future, - stderrDone.future, - ]); - await process.exitCode; - - } - - // Start `flutter run` again to make sure it loads from the previous - // state. dev compilers loads up from previously compiled JavaScript. - { - final Process process = await startProcess( - path.join(flutterDirectory.path, 'bin', 'flutter'), - flutterCommandArgs('run', options), - environment: { - 'FLUTTER_WEB': 'true', - }, - ); - final Completer stdoutDone = Completer(); - final Completer stderrDone = Completer(); - process.stdout - .transform(utf8.decoder) - .transform(const LineSplitter()) - .listen((String line) { - if (line.contains('To hot restart')) { - process.stdin.write('R'); - } - if (line.contains('Restarted')) { + final Completer stdoutDone = Completer(); + final Completer stderrDone = Completer(); + final Stopwatch sw = Stopwatch()..start(); + process.stdout + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((String line) { + if (line.contains('To hot restart')) { + // measure clean start-up time. + sw.stop(); + measurements[kInitialStartupTime] = sw.elapsedMilliseconds; + sw + ..reset() + ..start(); + process.stdin.write('R'); + } + if (line.contains('Recompile complete')) { + if (hotRestartCount == 0) { + measurements[kFirstRestartTime] = sw.elapsedMilliseconds; + // Update the file and reload again. + final File appDartSource = file(path.join( + _editedFlutterGalleryDir.path, 'lib/gallery/app.dart', + )); + appDartSource.writeAsStringSync( + appDartSource.readAsStringSync().replaceFirst( + "'Flutter Gallery'", "'Updated Flutter Gallery'", + ) + ); + sw + ..reset() + ..start(); + process.stdin.writeln('R'); + ++hotRestartCount; + } else { + measurements[kFirstRecompileTime] = sw.elapsedMilliseconds; + // Quit after second hot restart. process.stdin.writeln('q'); } - print('stdout: $line'); - }, onDone: () { - stdoutDone.complete(); - }); - process.stderr - .transform(utf8.decoder) - .transform(const LineSplitter()) - .listen((String line) { - print('stderr: $line'); - }, onDone: () { - stderrDone.complete(); - }); + } + print('stdout: $line'); + }, onDone: () { + stdoutDone.complete(); + }); + process.stderr + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((String line) { + print('stderr: $line'); + }, onDone: () { + stderrDone.complete(); + }); - await Future.wait(>[ - stdoutDone.future, - stderrDone.future, - ]); - await process.exitCode; - } - }); + await Future.wait(>[ + stdoutDone.future, + stderrDone.future, + ]); + await process.exitCode; + + } + + // Start `flutter run` again to make sure it loads from the previous + // state. dev compilers loads up from previously compiled JavaScript. + { + + final Stopwatch sw = Stopwatch()..start(); + final Process process = await startProcess( + path.join(flutterDirectory.path, 'bin', 'flutter'), + flutterCommandArgs('run', options), + environment: { + 'FLUTTER_WEB': 'true', + }, + ); + final Completer stdoutDone = Completer(); + final Completer stderrDone = Completer(); + process.stdout + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((String line) { + + if (line.contains('To hot restart')) { + measurements[kSecondStartupTime] = sw.elapsedMilliseconds; + sw + ..reset() + ..start(); + process.stdin.write('R'); + } + if (line.contains('Recompile complete')) { + measurements[kSecondRestartTime] = sw.elapsedMilliseconds; + process.stdin.writeln('q'); + } + print('stdout: $line'); + }, onDone: () { + stdoutDone.complete(); + }); + process.stderr + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((String line) { + print('stderr: $line'); + }, onDone: () { + stderrDone.complete(); + }); + + await Future.wait(>[ + stdoutDone.future, + stderrDone.future, + ]); + await process.exitCode; + } }); - } finally { - final Set afterChromeProcesses = await getRunningProcesses(processName: chromeProcessName) - .map((RunningProcessInfo info) => info.pid) - .toSet(); - final Set newProcesses = afterChromeProcesses.difference(beforeChromeProcesses); - for (String processId in newProcesses) { - await killProcess(processId); - } - } + }); if (hotRestartCount != 1) { return TaskResult.failure(null); } - return TaskResult.success(null); + return TaskResult.success(measurements, benchmarkScoreKeys: [ + kInitialStartupTime, + kFirstRestartTime, + kFirstRecompileTime, + kSecondStartupTime, + kSecondRestartTime, + ]); }; } diff --git a/dev/devicelab/manifest.yaml b/dev/devicelab/manifest.yaml index 0fb8d472600..3aefda71bda 100644 --- a/dev/devicelab/manifest.yaml +++ b/dev/devicelab/manifest.yaml @@ -126,13 +126,12 @@ tasks: stage: devicelab_win required_agent_capabilities: ["windows/android"] - # TODO(jonahwilliams): re-enable with https://github.com/flutter/flutter/issues/39597 - # windows_chrome_dev_mode: - # description: > - # Run flutter web on the devicelab and hot restart. - # stage: devicelab_win - # required_agent_capabilities: ["windows/android"] - # flaky: true + windows_chrome_dev_mode: + description: > + Run flutter web on the devicelab and hot restart. + stage: devicelab_win + required_agent_capabilities: ["windows/android"] + flaky: true # Android on-device tests @@ -346,13 +345,12 @@ tasks: stage: devicelab required_agent_capabilities: ["linux/android"] - # TODO(jonahwilliams): re-enable with https://github.com/flutter/flutter/issues/39597 - # linux_chrome_dev_mode: - # description: > - # Run flutter web on the devicelab and hot restart. - # stage: devicelab - # required_agent_capabilities: ["linux/android"] - # flaky: true + linux_chrome_dev_mode: + description: > + Run flutter web on the devicelab and hot restart. + stage: devicelab + required_agent_capabilities: ["linux/android"] + flaky: true web_size__compile_test: description: > @@ -519,12 +517,12 @@ tasks: stage: devicelab_ios required_agent_capabilities: ["mac/ios"] - # macos_chrome_dev_mode: - # description: > - # Run flutter web on the devicelab and hot restart. - # stage: devicelab_ios - # required_agent_capabilities: ["mac/ios"] - # flaky: true # marked as flaky while infra is under heavy development + macos_chrome_dev_mode: + description: > + Run flutter web on the devicelab and hot restart. + stage: devicelab_ios + required_agent_capabilities: ["mac/ios"] + flaky: true # marked as flaky while infra is under heavy development build_benchmark_ios: description: >