flutter_flutter/dev/devicelab/lib/tasks/flutter_tool_startup.dart

// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:io';

import 'package:path/path.dart' as path;

import '../framework/task_result.dart';
import '../framework/utils.dart';

/// Run each benchmark this many times and compute average, min, max.
const int _kRunsPerBenchmark = 10;

Future<TaskResult> flutterToolStartupBenchmarkTask() async {
  final Directory projectParentDirectory = Directory.systemTemp.createTempSync(
    'flutter_tool_startup_benchmark',
  );
  final Directory projectDirectory = dir(path.join(projectParentDirectory.path, 'benchmark'));
  await inDirectory<void>(flutterDirectory, () async {
    await flutter('update-packages');
    await flutter('create', options: <String>[projectDirectory.path]);
    // Remove 'test' directory so we don't time the actual testing, but only the launching of the flutter tool
    rmTree(dir(path.join(projectDirectory.path, 'test')));
  });

  final Map<String, dynamic> data = <String, dynamic>{
    // `flutter test` in dir with no `test` folder.
    ...(await _Benchmark(
      projectDirectory,
      'test startup',
      'test',
    ).run()).asMap('flutter_tool_startup_test'),

    // `flutter test -d foo_device` in dir with no `test` folder.
    ...(await _Benchmark(
      projectDirectory,
      'test startup with specified device',
      'test',
      options: <String>['-d', 'foo_device'],
    ).run()).asMap('flutter_tool_startup_test_with_specified_device'),

    // `flutter test -v` where no android sdk will be found (at least currently).
    ...(await _Benchmark(
      projectDirectory,
      'test startup no android sdk',
      'test',
      options: <String>['-v'],
      environment: <String, String>{
        'ANDROID_HOME': 'dummy value',
        'ANDROID_SDK_ROOT': 'dummy value',
        'PATH': pathWithoutWhereHits(<String>['adb', 'aapt']),
      },
    ).run()).asMap('flutter_tool_startup_test_no_android_sdk'),

    // `flutter -h`.
    ...(await _Benchmark(
      projectDirectory,
      'help startup',
      '-h',
    ).run()).asMap('flutter_tool_startup_help'),
  };

  // Cleanup.
  rmTree(projectParentDirectory);

  return TaskResult.success(data, benchmarkScoreKeys: data.keys.toList());
}

String pathWithoutWhereHits(List<String> whats) {
  final String pathEnvironment = Platform.environment['PATH'] ?? '';
  List<String> paths;
  if (Platform.isWindows) {
    paths = pathEnvironment.split(';');
  } else {
    paths = pathEnvironment.split(':');
  }
  // This isn't great but will probably work for our purposes.
  final List<String> extensions = <String>['', '.exe', '.bat', '.com'];

  final List<String> notFound = <String>[];
  for (final String path in paths) {
    bool found = false;
    for (final String extension in extensions) {
      for (final String what in whats) {
        final File f = File('$path${Platform.pathSeparator}$what$extension');
        if (f.existsSync()) {
          found = true;
          break;
        }
      }
      if (found) {
        break;
      }
    }
    if (!found) {
      notFound.add(path);
    }
  }

  if (Platform.isWindows) {
    return notFound.join(';');
  } else {
    return notFound.join(':');
  }
}

class _BenchmarkResult {
  const _BenchmarkResult(this.mean, this.min, this.max);

  final int mean; // Milliseconds

  final int min; // Milliseconds

  final int max; // Milliseconds

  Map<String, dynamic> asMap(String name) {
    return <String, dynamic>{name: mean, '${name}_minimum': min, '${name}_maximum': max};
  }
}

class _Benchmark {
  _Benchmark(
    this.directory,
    this.title,
    this.command, {
    this.options = const <String>[],
    this.environment,
  });

  final Directory directory;

  final String title;

  final String command;

  final List<String> options;

  final Map<String, String>? environment;

  Future<int> execute(int iteration, int targetIterations) async {
    section('Benchmark $title - ${iteration + 1} / $targetIterations');
    final Stopwatch stopwatch = Stopwatch();
    await inDirectory<void>(directory, () async {
      stopwatch.start();
      // canFail is set to true, as e.g. `flutter test` in a dir with no `test`
      // directory sets a non-zero return value.
      await flutter(command, options: options, canFail: true, environment: environment);
      stopwatch.stop();
    });
    return stopwatch.elapsedMilliseconds;
  }

  /// Runs `benchmark` several times and reports the results.
  Future<_BenchmarkResult> run() async {
    final List<int> results = <int>[];
    int sum = 0;
    for (int i = 0; i < _kRunsPerBenchmark; i++) {
      final int thisRuntime = await execute(i, _kRunsPerBenchmark);
      results.add(thisRuntime);
      sum += thisRuntime;
    }
    results.sort();
    return _BenchmarkResult(sum ~/ results.length, results.first, results.last);
  }
}