flutter_flutter/dev/bots/suite_runners/run_test_harness_tests.dart

// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:convert';
import 'dart:io' show File, Platform;

import 'package:path/path.dart' as path;

import '../run_command.dart';
import '../utils.dart';

String get platformFolderName {
  if (Platform.isWindows) {
    return 'windows-x64';
  }
  if (Platform.isMacOS) {
    return 'darwin-x64';
  }
  if (Platform.isLinux) {
    return 'linux-x64';
  }
  throw UnsupportedError(
    'The platform ${Platform.operatingSystem} is not supported by this script.',
  );
}

Future<void> testHarnessTestsRunner() async {
  printProgress('${green}Running test harness tests...$reset');

  await _validateEngineRevision();

  // Verify that the tests actually return failure on failure and success on
  // success.
  final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests');

  // We want to run these tests in parallel, because they each take some time
  // to run (e.g. compiling), so we don't want to run them in series, especially
  // on 20-core machines. However, we have a race condition, so for now...
  // Race condition issue: https://github.com/flutter/flutter/issues/90026
  final tests = <ShardRunner>[
    () => runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'pass_test.dart'),
      printOutput: false,
    ),
    () => runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'fail_test.dart'),
      expectFailure: true,
      printOutput: false,
    ),
    () => runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'pending_timer_fail_test.dart'),
      expectFailure: true,
      printOutput: false,
      outputChecker: (CommandResult result) {
        return result.flattenedStdout!.contains('failingPendingTimerTest')
            ? null
            : 'Failed to find the stack trace for the pending Timer.\n\n'
                  'stdout:\n${result.flattenedStdout}\n\n'
                  'stderr:\n${result.flattenedStderr}';
      },
    ),
    () => runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'fail_test_on_exception_after_test.dart'),
      expectFailure: true,
      printOutput: false,
      outputChecker: (CommandResult result) {
        const expectedError =
            '══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞════════════════════════════════════════════════════\n'
            'The following StateError was thrown running a test (but after the test had completed):\n'
            'Bad state: Exception thrown after test completed.';
        if (result.flattenedStdout!.contains(expectedError)) {
          return null;
        }
        return 'Failed to find expected output on stdout.\n\n'
            'Expected output:\n$expectedError\n\n'
            'Actual stdout:\n${result.flattenedStdout}\n\n'
            'Actual stderr:\n${result.flattenedStderr}';
      },
    ),
    () => runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'crash1_test.dart'),
      expectFailure: true,
      printOutput: false,
    ),
    () => runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'crash2_test.dart'),
      expectFailure: true,
      printOutput: false,
    ),
    () => runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'syntax_error_test.broken_dart'),
      expectFailure: true,
      printOutput: false,
    ),
    () => runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'missing_import_test.broken_dart'),
      expectFailure: true,
      printOutput: false,
    ),
    () => runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'disallow_error_reporter_modification_test.dart'),
      expectFailure: true,
      printOutput: false,
    ),
  ];

  List<ShardRunner> testsToRun;

  // Run all tests unless sharding is explicitly specified.
  final String? shardName = Platform.environment[kShardKey];
  if (shardName == kTestHarnessShardName) {
    testsToRun = selectIndexOfTotalSubshard<ShardRunner>(tests);
  } else {
    testsToRun = tests;
  }
  for (final test in testsToRun) {
    await test();
  }

  // Verify that we correctly generated the version file.
  if (await Version.resolveIn() case final VersionError e) {
    foundError(<String>[e.error]);
  }
}

/// Verify the Flutter Engine is the revision in `bin/cache/engine_stamp.json` key: git_revision.
Future<void> _validateEngineRevision() async {
  final String flutterTester = path.join(
    flutterRoot,
    'bin',
    'cache',
    'artifacts',
    'engine',
    platformFolderName,
    'flutter_tester$exe',
  );

  // TODO(matanlurey): Revisit with the Dart team if this is true now that they use FLUTTER_PREBUILT_ENGINE_VERSION=...
  if (runningInDartHHHBot) {
    // The Dart HHH bots intentionally modify the local artifact cache
    // and then use this script to run Flutter's test suites.
    // Because the artifacts have been changed, this particular test will return
    // a false positive and should be skipped.
    print('${yellow}Skipping Flutter Engine Version Validation for swarming bot $luciBotId.');
    return;
  }

  final String expectedVersion;
  if (json.decode(File(engineInfoFile).readAsStringSync().trim()) as Map<String, Object?> case {
    'git_revision': final String parsedVersion,
  }) {
    expectedVersion = parsedVersion;
  } else {
    throw 'engine_stamp.json missing "git_revision" key';
  }

  final CommandResult result = await runCommand(flutterTester, <String>[
    '--help',
  ], outputMode: OutputMode.capture);
  if (result.flattenedStdout!.isNotEmpty) {
    foundError(<String>[
      '${red}The stdout of `$flutterTester --help` was not empty:$reset',
      ...result.flattenedStdout!.split('\n').map((String line) => ' $gray┆$reset $line'),
    ]);
  }
  final String actualVersion;
  try {
    actualVersion = result.flattenedStderr!.split('\n').firstWhere((final String line) {
      return line.startsWith('Flutter Engine Version:');
    });
  } on StateError {
    foundError(<String>[
      '${red}Could not find "Flutter Engine Version:" line in `${path.basename(flutterTester)} --help` stderr output:$reset',
      ...result.flattenedStderr!.split('\n').map((String line) => ' $gray┆$reset $line'),
    ]);
    return;
  }
  if (!actualVersion.contains(expectedVersion)) {
    foundError(<String>[
      '${red}Expected "Flutter Engine Version: $expectedVersion", but found "$actualVersion".$reset',
    ]);
  }
}