A minimal engine_tools_lib to use for local-repo Dart tooling (flutter/engine#45154)

Partial work towards re-landing #44936.

Both the `clang_tidy` and `githooks` passage could benefit from being
able to automatically find the latest `compile_commands.json` output,
which means that some common code should exist in the `tools/`
directory.

This is a very minimal (but tested) library for doing exactly that.

engine/src/flutter/testing/run_tests.py

@@ -953,6 +953,25 @@ def gather_clang_tidy_tests(build_dir):
     )
 
 
+def gather_engine_repo_tools_tests(build_dir):
+  test_dir = os.path.join(
+      BUILDROOT_DIR, 'flutter', 'tools', 'pkg', 'engine_repo_tools'
+  )
+  dart_tests = glob.glob('%s/*_test.dart' % test_dir)
+  for dart_test_file in dart_tests:
+    opts = [
+        '--disable-dart-dev',
+        dart_test_file,
+    ]
+    yield EngineExecutableTask(
+        build_dir,
+        os.path.join('dart-sdk', 'bin', 'dart'),
+        None,
+        flags=opts,
+        cwd=test_dir
+    )
+
+
 def gather_api_consistency_tests(build_dir):
   test_dir = os.path.join(BUILDROOT_DIR, 'flutter', 'tools', 'api_check')
   dart_tests = glob.glob('%s/test/*_test.dart' % test_dir)
@@ -1230,6 +1249,7 @@ Flutter Wiki page on the subject: https://github.com/flutter/flutter/wiki/Testin
     tasks += list(gather_litetest_tests(build_dir))
     tasks += list(gather_githooks_tests(build_dir))
     tasks += list(gather_clang_tidy_tests(build_dir))
+    tasks += list(gather_engine_repo_tools_tests(build_dir))
     tasks += list(gather_api_consistency_tests(build_dir))
     tasks += list(gather_path_ops_tests(build_dir))
     tasks += list(gather_const_finder_tests(build_dir))

engine/src/flutter/tools/pkg/engine_repo_tools/README.md

@@ -0,0 +1,17 @@
+# engine_repo_tools
+
+This is a repo-internal library for `flutter/engine`, that contains shared code
+for writing tools that operate on the engine repository. For example, finding
+the latest compiled engine artifacts in the `out/` directory:
+
+```dart
+import 'package:engine_repo_tools/engine_repo_tools.dart';
+
+void main() {
+  final engine = Engine.findWithin();
+  final latest = engine.latestOutput();
+  if (latest != null) {
+    print('Latest compile_commands.json: ${latest.compileCommandsJson?.path}');
+  }
+}
+```

engine/src/flutter/tools/pkg/engine_repo_tools/lib/engine_repo_tools.dart

@@ -0,0 +1,228 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+/// A minimal library for discovering and probing a local engine repository.
+///
+/// This library is intended to be used by tools that need to interact with a
+/// local engine repository, such as `clang_tidy` or `githooks`. For example,
+/// finding the `compile_commands.json` file for the most recently built output:
+///
+/// ```dart
+/// final Engine engine = Engine.findWithin();
+/// final Output? output = engine.latestOutput();
+/// if (output == null) {
+///   print('No output targets found.');
+/// } else {
+///   final io.File? compileCommandsJson = output.compileCommandsJson;
+///   if (compileCommandsJson == null) {
+///     print('No compile_commands.json file found.');
+///   } else {
+///     print('Found compile_commands.json file at ${compileCommandsJson.path}');
+///   }
+/// }
+/// ```
+library;
+
+import 'dart:io' as io;
+
+import 'package:path/path.dart' as p;
+
+/// Represents the `$ENGINE` directory (i.e. a checked-out Flutter engine).
+///
+/// If you have a path to the `$ENGINE/src` directory, use [Engine.fromSrcPath].
+///
+/// If you have a path to a directory within the `$ENGINE/src` directory, or
+/// want to use the current working directory, use [Engine.findWithin].
+final class Engine {
+  /// Creates an [Engine] from a path such as `/Users/.../flutter/engine/src`.
+  ///
+  /// ```dart
+  /// final Engine engine = Engine.findWithin('/Users/.../engine/src');
+  /// print(engine.srcDir.path); // /Users/.../engine/src
+  /// ```
+  ///
+  /// Throws a [InvalidEngineException] if the path is not a valid engine root.
+  factory Engine.fromSrcPath(String srcPath) {
+    // If the path does not end in `/src`, fail.
+    if (p.basename(srcPath) != 'src') {
+      throw InvalidEngineException.doesNotEndWithSrc(srcPath);
+    }
+
+    // If the directory does not exist, or is not a directory, fail.
+    final io.Directory srcDir = io.Directory(srcPath);
+    if (!srcDir.existsSync()) {
+      throw InvalidEngineException.notADirectory(srcPath);
+    }
+
+    // Check for the existence of a `flutter` directory within `src`.
+    final io.Directory flutterDir = io.Directory(p.join(srcPath, 'flutter'));
+    if (!flutterDir.existsSync()) {
+      throw InvalidEngineException.missingFlutterDirectory(srcPath);
+    }
+
+    // We do **NOT** check for the existence of a `out` directory within `src`,
+    // it's not required to exist (i.e. a new checkout of the engine), and we
+    // don't want to fail if it doesn't exist.
+    final io.Directory outDir = io.Directory(p.join(srcPath, 'out'));
+
+    return Engine._(srcDir, flutterDir, outDir);
+  }
+
+  /// Creates an [Engine] by looking for a `src/` directory in the given path.
+  ///
+  /// ```dart
+  /// // Use the current working directory.
+  /// final Engine engine = Engine.findWithin();
+  /// print(engine.srcDir.path); // /Users/.../engine/src
+  ///
+  /// // Use a specific directory.
+  /// final Engine engine = Engine.findWithin('/Users/.../engine/src/foo/bar/baz');
+  /// print(engine.srcDir.path); // /Users/.../engine/src
+  /// ```
+  ///
+  /// If a path is not provided, the current working directory is used.
+  ///
+  /// Throws a [StateError] if the path is not within a valid engine.
+  factory Engine.findWithin([String? path]) {
+    path ??= p.current;
+
+    // Search parent directories for a `src` directory.
+    io.Directory maybeSrcDir = io.Directory(path);
+
+    if (!maybeSrcDir.existsSync()) {
+      throw StateError(
+        'The path "$path" does not exist or is not a directory.'
+      );
+    }
+
+    do {
+      try {
+        return Engine.fromSrcPath(maybeSrcDir.path);
+      } on InvalidEngineException {
+        // Ignore, we'll keep searching.
+      }
+      maybeSrcDir = maybeSrcDir.parent;
+    } while (maybeSrcDir.parent.path != maybeSrcDir.path /* at root */);
+
+    throw StateError(
+      'The path "$path" is not within a Flutter engine source directory.'
+    );
+  }
+
+  const Engine._(
+    this.srcDir,
+    this.flutterDir,
+    this.outDir,
+  );
+
+  /// The path to the `$ENGINE/src` directory.
+  final io.Directory srcDir;
+
+  /// The path to the `$ENGINE/src/flutter` directory.
+  final io.Directory flutterDir;
+
+  /// The path to the `$ENGINE/src/out` directory.
+  ///
+  /// **NOTE**: This directory may not exist.
+  final io.Directory outDir;
+
+  /// Returns a list of all output targets in [outDir].
+  List<Output> outputs() {
+    return outDir
+      .listSync()
+      .whereType<io.Directory>()
+      .map<Output>(Output._)
+      .toList();
+  }
+
+  /// Returns the most recently modified output target in [outDir].
+  ///
+  /// If there are no output targets, returns `null`.
+  Output? latestOutput() {
+    final List<Output> outputs = this.outputs();
+    if (outputs.isEmpty) {
+      return null;
+    }
+    outputs.sort((Output a, Output b) {
+      return b.dir.statSync().modified.compareTo(a.dir.statSync().modified);
+    });
+    return outputs.first;
+  }
+}
+
+/// Thrown when an [Engine] could not be created from a path.
+sealed class InvalidEngineException implements Exception {
+  /// Thrown when an [Engine] was created from a path not ending in `src`.
+  factory InvalidEngineException.doesNotEndWithSrc(String path) {
+    return InvalidEngineSrcPathException._(path);
+  }
+
+  /// Thrown when an [Engine] was created from a directory that does not exist.
+  factory InvalidEngineException.notADirectory(String path) {
+    return InvalidEngineNotADirectoryException._(path);
+  }
+
+  /// Thrown when an [Engine] was created from a path not containing `flutter/`.
+  factory InvalidEngineException.missingFlutterDirectory(String path) {
+    return InvalidEngineMissingFlutterDirectoryException._(path);
+  }
+}
+
+/// Thrown when an [Engine] was created from a path not ending in `src`.
+final class InvalidEngineSrcPathException implements InvalidEngineException {
+  InvalidEngineSrcPathException._(this.path);
+
+  /// The path that was used to create the [Engine].
+  final String path;
+
+  @override
+  String toString() {
+    return 'The path $path does not end in `${p.separator}src`.';
+  }
+}
+
+/// Thrown when an [Engine] was created from a path that is not a directory.
+final class InvalidEngineNotADirectoryException implements InvalidEngineException {
+  InvalidEngineNotADirectoryException._(this.path);
+
+  /// The path that was used to create the [Engine].
+  final String path;
+
+  @override
+  String toString() {
+    return 'The path "$path" does not exist or is not a directory.';
+  }
+}
+
+/// Thrown when an [Engine] was created from a path not containing `flutter/`.
+final class InvalidEngineMissingFlutterDirectoryException implements InvalidEngineException {
+  InvalidEngineMissingFlutterDirectoryException._(this.path);
+
+  /// The path that was used to create the [Engine].
+  final String path;
+
+  @override
+  String toString() {
+    return 'The path "$path" does not contain a "flutter" directory.';
+  }
+}
+
+/// Represents a single output target in the `$ENGINE/src/out` directory.
+final class Output {
+  const Output._(this.dir);
+
+  /// The directory containing the output target.
+  final io.Directory dir;
+
+  /// The `compile_commands.json` file for this output target.
+  ///
+  /// Returns `null` if the file does not exist.
+  io.File? get compileCommandsJson {
+    final io.File file = io.File(p.join(dir.path, 'compile_commands.json'));
+    if (!file.existsSync()) {
+      return null;
+    }
+    return file;
+  }
+}

engine/src/flutter/tools/pkg/engine_repo_tools/pubspec.yaml

@@ -0,0 +1,41 @@
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+name: engine_repo_tools
+publish_to: none
+environment:
+  sdk: ^3.0.0
+
+# Do not add any dependencies that require more than what is provided in
+# //third_party/pkg, //third_party/dart/pkg, or
+# //third_party/dart/third_party/pkg. In particular, package:test is not usable
+# here.
+
+# If you do add packages here, make sure you can run `pub get --offline`, and
+# check the .packages and .package_config to make sure all the paths are
+# relative to this directory into //third_party/dart
+
+dependencies:
+  meta: any
+  path: any
+
+dev_dependencies:
+  async_helper: any
+  expect: any
+  litetest: any
+  smith: any
+
+dependency_overrides:
+  async_helper:
+    path: ../../../../third_party/dart/pkg/async_helper
+  expect:
+    path: ../../../../third_party/dart/pkg/expect
+  litetest:
+    path: ../../../testing/litetest
+  meta:
+    path: ../../../../third_party/dart/pkg/meta
+  path:
+    path: ../../../../third_party/dart/third_party/pkg/path
+  smith:
+    path: ../../../../third_party/dart/pkg/smith

engine/src/flutter/tools/pkg/engine_repo_tools/test/engine_repo_tools_test.dart

@@ -0,0 +1,257 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io' as io;
+import 'package:async_helper/async_helper.dart';
+import 'package:engine_repo_tools/engine_repo_tools.dart';
+import 'package:litetest/litetest.dart';
+import 'package:path/path.dart' as p;
+
+void main() {
+  late io.Directory emptyDir;
+
+  void setUp() {
+    emptyDir = io.Directory.systemTemp.createTempSync('engine_repo_tools.test');
+  }
+
+  void tearDown() {
+    emptyDir.deleteSync(recursive: true);
+  }
+
+  group('Engine.fromSrcPath', () {
+    group('should fail when', () {
+      test('the path does not end in `${p.separator}src`', () {
+        setUp();
+        try {
+          expect(
+          () => Engine.fromSrcPath(emptyDir.path),
+          _throwsInvalidEngineException,
+        );
+        } finally {
+          tearDown();
+        }
+      });
+
+      test('the path does not exist', () {
+        setUp();
+        try {
+          expect(
+            () => Engine.fromSrcPath(p.join(emptyDir.path, 'src')),
+            _throwsInvalidEngineException,
+          );
+        } finally {
+          tearDown();
+        }
+      });
+
+      test('the path does not contain a "flutter" directory', () {
+        setUp();
+        try {
+          final io.Directory srcDir = io.Directory(p.join(emptyDir.path, 'src'))..createSync();
+          expect(
+            () => Engine.fromSrcPath(srcDir.path),
+            _throwsInvalidEngineException,
+          );
+        } finally {
+          tearDown();
+        }
+      });
+
+      test('returns an Engine', () {
+        setUp();
+        try {
+          final io.Directory srcDir = io.Directory(p.join(emptyDir.path, 'src'))..createSync();
+          io.Directory(p.join(srcDir.path, 'flutter')).createSync();
+          io.Directory(p.join(srcDir.path, 'out')).createSync();
+
+          final Engine engine = Engine.fromSrcPath(srcDir.path);
+
+          expect(engine.srcDir.path, srcDir.path);
+          expect(engine.flutterDir.path, p.join(srcDir.path, 'flutter'));
+          expect(engine.outDir.path, p.join(srcDir.path, 'out'));
+        } finally {
+          tearDown();
+        }
+      });
+    });
+  });
+
+  group('Engine.findWithin', () {
+    late io.Directory emptyDir;
+
+    void setUp() {
+      emptyDir = io.Directory.systemTemp.createTempSync('engine_repo_tools.test');
+    }
+
+    void tearDown() {
+      emptyDir.deleteSync(recursive: true);
+    }
+
+    group('should fail when', () {
+      test('the path does not contain a "src" directory', () {
+        setUp();
+        try {
+          expect(
+            () => Engine.findWithin(emptyDir.path),
+            throwsStateError,
+          );
+        } finally {
+          tearDown();
+        }
+      });
+
+      test('the path contains a "src" directory but it is not an engine root', () {
+        setUp();
+        try {
+          final io.Directory srcDir = io.Directory(p.join(emptyDir.path, 'src'))..createSync();
+          expect(
+            () => Engine.findWithin(srcDir.path),
+            throwsStateError,
+          );
+        } finally {
+          tearDown();
+        }
+      });
+
+      test('returns an Engine', () {
+        setUp();
+        try {
+          final io.Directory srcDir = io.Directory(p.join(emptyDir.path, 'src'))..createSync();
+          io.Directory(p.join(srcDir.path, 'flutter')).createSync();
+          io.Directory(p.join(srcDir.path, 'out')).createSync();
+
+          final Engine engine = Engine.findWithin(srcDir.path);
+
+          expect(engine.srcDir.path, srcDir.path);
+          expect(engine.flutterDir.path, p.join(srcDir.path, 'flutter'));
+          expect(engine.outDir.path, p.join(srcDir.path, 'out'));
+        } finally {
+          tearDown();
+        }
+      });
+
+      test('returns an Engine even if a "src" directory exists deeper in the tree', () {
+        // It's common to have "src" directories, so if we have something like:
+        //  /Users/.../engine/src/foo/bar/src/baz
+        //
+        // And we use `Engine.findWithin('/Users/.../engine/src/flutter/bar/src/baz')`,
+        // we should still find the engine (in this case, the engine root is
+        // `/Users/.../engine/src`).
+        setUp();
+        try {
+          final io.Directory srcDir = io.Directory(p.join(emptyDir.path, 'src'))..createSync();
+          io.Directory(p.join(srcDir.path, 'flutter')).createSync();
+          io.Directory(p.join(srcDir.path, 'out')).createSync();
+
+          final io.Directory nestedSrcDir = io.Directory(p.join(srcDir.path, 'flutter', 'bar', 'src', 'baz'))..createSync(recursive: true);
+
+          final Engine engine = Engine.findWithin(nestedSrcDir.path);
+
+          expect(engine.srcDir.path, srcDir.path);
+          expect(engine.flutterDir.path, p.join(srcDir.path, 'flutter'));
+          expect(engine.outDir.path, p.join(srcDir.path, 'out'));
+        } finally {
+          tearDown();
+        }
+      });
+    });
+  });
+
+  test('outputs an empty list of targets', () {
+    setUp();
+
+    try {
+      // Create a valid engine.
+      io.Directory(p.join(emptyDir.path, 'src', 'flutter')).createSync(recursive: true);
+      io.Directory(p.join(emptyDir.path, 'src', 'out')).createSync(recursive: true);
+
+      final Engine engine = Engine.fromSrcPath(p.join(emptyDir.path, 'src'));
+      expect(engine.outputs(), <Output>[]);
+      expect(engine.latestOutput(), isNull);
+    } finally {
+      tearDown();
+    }
+  });
+
+  test('outputs a list of targets', () {
+    setUp();
+
+    try {
+      // Create a valid engine.
+      io.Directory(p.join(emptyDir.path, 'src', 'flutter')).createSync(recursive: true);
+      io.Directory(p.join(emptyDir.path, 'src', 'out')).createSync(recursive: true);
+
+      // Create two targets in out: host_debug and host_debug_unopt_arm64.
+      io.Directory(p.join(emptyDir.path, 'src', 'out', 'host_debug')).createSync(recursive: true);
+      io.Directory(p.join(emptyDir.path, 'src', 'out', 'host_debug_unopt_arm64')).createSync(recursive: true);
+
+      final Engine engine = Engine.fromSrcPath(p.join(emptyDir.path, 'src'));
+      final List<String> outputs = engine.outputs().map((Output o) => p.basename(o.dir.path)).toList()..sort();
+      expect(outputs, <String>[
+        'host_debug',
+        'host_debug_unopt_arm64',
+      ]);
+    } finally {
+      tearDown();
+    }
+  });
+
+  test('outputs the latest target and compile_commands.json', () {
+    setUp();
+
+    try {
+      // Create a valid engine.
+      io.Directory(p.join(emptyDir.path, 'src', 'flutter')).createSync(recursive: true);
+      io.Directory(p.join(emptyDir.path, 'src', 'out')).createSync(recursive: true);
+
+      // Create two targets in out: host_debug and host_debug_unopt_arm64.
+      io.Directory(p.join(emptyDir.path, 'src', 'out', 'host_debug')).createSync(recursive: true);
+      io.Directory(p.join(emptyDir.path, 'src', 'out', 'host_debug_unopt_arm64')).createSync(recursive: true);
+
+      // Intentionnally make host_debug a day old to ensure it is not picked.
+      final io.File oldJson = io.File(p.join(emptyDir.path, 'src', 'out', 'host_debug', 'compile_commands.json'))..createSync();
+      oldJson.setLastModifiedSync(oldJson.lastModifiedSync().subtract(const Duration(days: 1)));
+
+      io.File(p.join(emptyDir.path, 'src', 'out', 'host_debug_unopt_arm64', 'compile_commands.json')).createSync();
+
+      final Engine engine = Engine.fromSrcPath(p.join(emptyDir.path, 'src'));
+      final Output? latestOutput = engine.latestOutput();
+      expect(latestOutput, isNotNull);
+      expect(p.basename(latestOutput!.dir.path), 'host_debug_unopt_arm64');
+      expect(latestOutput.compileCommandsJson, isNotNull);
+    } finally {
+      tearDown();
+    }
+  });
+}
+
+// This is needed because async_minitest and friends is not a proper testing
+// library and is missing a lot of functionality that was exclusively added
+// to pkg/test.
+void _throwsInvalidEngineException(Object? o) {
+  _checkThrow<InvalidEngineException>(o, (_){});
+}
+
+// Mostly copied from async_minitest.
+void _checkThrow<T extends Object>(dynamic v, void Function(dynamic error) onError) {
+  if (v is Future) {
+    asyncStart();
+    v.then((_) {
+      Expect.fail('Did not throw');
+    }, onError: (Object e, StackTrace s) {
+      if (e is! T) {
+        // ignore: only_throw_errors
+        throw e;
+      }
+      onError(e);
+      asyncEnd();
+    });
+    return;
+  }
+  v as void Function();
+  Expect.throws<T>(v, (T e) {
+    onError(e);
+    return true;
+  });
+}

engine/src/flutter/tools/pub_get_offline.py

@@ -35,6 +35,7 @@ ALL_PACKAGES = [
     os.path.join(ENGINE_DIR, "tools", "android_lint"),
     os.path.join(ENGINE_DIR, "tools", "clang_tidy"),
     os.path.join(ENGINE_DIR, "tools", "const_finder"),
+    os.path.join(ENGINE_DIR, "tools", "pkg", "engine_repo_tools"),
     os.path.join(ENGINE_DIR, "tools", "githooks"),
     os.path.join(ENGINE_DIR, "tools", "licenses"),
     os.path.join(ENGINE_DIR, "tools", "path_ops", "dart")