From 4c47fdadd4ea8cb7176ac0f79647f1122e63b1c6 Mon Sep 17 00:00:00 2001 From: Jonah Williams Date: Fri, 25 Oct 2019 15:03:13 -0700 Subject: [PATCH] Add devfs for incremental compiler JavaScript bundle (#43219) --- packages/flutter_tools/lib/src/base/io.dart | 1 + .../flutter_tools/lib/src/web/devfs_web.dart | 160 +++++++++++ .../general.shard/web/devfs_web_test.dart | 266 ++++++++++++++++++ 3 files changed, 427 insertions(+) create mode 100644 packages/flutter_tools/lib/src/web/devfs_web.dart create mode 100644 packages/flutter_tools/test/general.shard/web/devfs_web_test.dart diff --git a/packages/flutter_tools/lib/src/base/io.dart b/packages/flutter_tools/lib/src/base/io.dart index b7e8d724b5c..2282d81fa9e 100644 --- a/packages/flutter_tools/lib/src/base/io.dart +++ b/packages/flutter_tools/lib/src/base/io.dart @@ -52,6 +52,7 @@ export 'dart:io' HttpException, HttpHeaders, HttpRequest, + HttpResponse, HttpServer, HttpStatus, InternetAddress, diff --git a/packages/flutter_tools/lib/src/web/devfs_web.dart b/packages/flutter_tools/lib/src/web/devfs_web.dart new file mode 100644 index 00000000000..a28682fd4c7 --- /dev/null +++ b/packages/flutter_tools/lib/src/web/devfs_web.dart @@ -0,0 +1,160 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:typed_data'; + +import 'package:meta/meta.dart'; +import 'package:mime/mime.dart' as mime; + +import '../base/common.dart'; +import '../base/file_system.dart'; +import '../base/io.dart'; +import '../build_info.dart'; +import '../convert.dart'; +import '../globals.dart'; + +/// A web server which handles serving JavaScript and assets. +/// +/// This is only used in development mode. +class WebAssetServer { + @visibleForTesting + WebAssetServer(this._httpServer, { @required void Function(dynamic, StackTrace) onError }) { + _httpServer.listen((HttpRequest request) { + _handleRequest(request).catchError(onError); + // TODO(jonahwilliams): test the onError callback when https://github.com/dart-lang/sdk/issues/39094 is fixed. + }, onError: onError); + } + + // Fallback to "application/octet-stream" on null which + // makes no claims as to the structure of the data. + static const String _kDefaultMimeType = 'application/octet-stream'; + + /// Start the web asset server on a [hostname] and [port]. + /// + /// Unhandled exceptions will throw a [ToolExit] with the error and stack + /// trace. + static Future start(String hostname, int port) async { + try { + final HttpServer httpServer = await HttpServer.bind(hostname, port); + return WebAssetServer(httpServer, onError: (dynamic error, StackTrace stackTrace) { + httpServer.close(force: true); + throwToolExit('Unhandled exception in web development server:\n$error\n$stackTrace'); + }); + } on SocketException catch (err) { + throwToolExit('Failed to bind web development server:\n$err'); + } + assert(false); + return null; + } + + final HttpServer _httpServer; + final Map _files = {}; + + // handle requests for JavaScript source, dart sources maps, or asset files. + Future _handleRequest(HttpRequest request) async { + final HttpResponse response = request.response; + // If the response is `/`, then we are requesting the index file. + if (request.uri.path == '/') { + final File indexFile = fs.currentDirectory + .childDirectory('web') + .childFile('index.html'); + if (indexFile.existsSync()) { + response.headers.add('Content-Type', 'text/html'); + response.headers.add('Content-Length', indexFile.lengthSync()); + await response.addStream(indexFile.openRead()); + } else { + response.statusCode = HttpStatus.notFound; + } + await response.close(); + return; + } + + // If this is a JavaScript file, it must be in the in-memory cache. + // Attempt to look up the file by URI, returning a 404 if it is not + // found. + if (_files.containsKey(request.uri.path)) { + final List bytes = _files[request.uri.path]; + response.headers + ..add('Content-Length', bytes.length) + ..add('Content-Type', 'application/javascript'); + response.add(bytes); + await response.close(); + return; + } + // If this is a dart file, it must be on the local file system and is + // likely coming from a source map request. Attempt to look in the + // local filesystem for it, and return a 404 if it is not found. The tool + // doesn't currently consider the case of Dart files as assets. + File file = fs.file(Uri.base.resolve(request.uri.path)); + + // If both of the lookups above failed, the file might have been an asset. + // Try and resolve the path relative to the built asset directory. + if (!file.existsSync()) { + final String assetPath = request.uri.path.replaceFirst('/assets/', ''); + file = fs.file(fs.path.join(getAssetBuildDirectory(), fs.path.relative(assetPath))); + } + + if (!file.existsSync()) { + response.statusCode = HttpStatus.notFound; + await response.close(); + return; + } + final int length = file.lengthSync(); + // Attempt to determine the file's mime type. if this is not provided some + // browsers will refuse to render images/show video et cetera. If the tool + // cannot determine a mime type, fall back to application/octet-stream. + String mimeType; + if (length >= 12) { + mimeType= mime.lookupMimeType( + file.path, + headerBytes: await file.openRead(0, 12).first, + ); + } + mimeType ??= _kDefaultMimeType; + response.headers.add('Content-Length', length); + response.headers.add('Content-Type', mimeType); + await response.addStream(file.openRead()); + await response.close(); + } + + /// Tear down the http server running. + Future dispose() { + return _httpServer.close(); + } + + /// Write a single file into the in-memory cache. + void writeFile(String filePath, String contents) { + _files[filePath] = Uint8List.fromList(utf8.encode(contents)); + } + + /// Update the in-memory asset server with the provided source and manifest files. + /// + /// Returns a list of updated modules. + List write(File sourceFile, File manifestFile) { + final List modules = []; + final Uint8List bytes = sourceFile.readAsBytesSync(); + final Map manifest = json.decode(manifestFile.readAsStringSync()); + for (String filePath in manifest.keys) { + if (filePath == null) { + printTrace('Invalid manfiest file: $filePath'); + continue; + } + final List offsets = manifest[filePath]; + if (offsets.length != 2) { + printTrace('Invalid manifest byte offsets: $offsets'); + continue; + } + final int start = offsets[0]; + final int end = offsets[1]; + if (start < 0 || end > bytes.lengthInBytes) { + printTrace('Invalid byte index: [$start, $end]'); + continue; + } + final Uint8List byteView = Uint8List.view(bytes.buffer, start, end - start); + _files[filePath] = byteView; + modules.add(filePath); + } + return modules; + } +} diff --git a/packages/flutter_tools/test/general.shard/web/devfs_web_test.dart b/packages/flutter_tools/test/general.shard/web/devfs_web_test.dart new file mode 100644 index 00000000000..44961c9c645 --- /dev/null +++ b/packages/flutter_tools/test/general.shard/web/devfs_web_test.dart @@ -0,0 +1,266 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:io'; + +import 'package:flutter_tools/src/base/common.dart'; +import 'package:flutter_tools/src/base/file_system.dart'; +import 'package:flutter_tools/src/base/io.dart'; +import 'package:flutter_tools/src/base/platform.dart'; +import 'package:flutter_tools/src/convert.dart'; +import 'package:flutter_tools/src/web/devfs_web.dart'; +import 'package:mockito/mockito.dart'; + +import '../../src/common.dart'; +import '../../src/testbed.dart'; + +const List kTransparentImage = [ + 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, + 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06, + 0x00, 0x00, 0x00, 0x1F, 0x15, 0xC4, 0x89, 0x00, 0x00, 0x00, 0x0A, 0x49, 0x44, + 0x41, 0x54, 0x78, 0x9C, 0x63, 0x00, 0x01, 0x00, 0x00, 0x05, 0x00, 0x01, 0x0D, + 0x0A, 0x2D, 0xB4, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E, 0x44, 0xAE, +]; + +void main() { + MockHttpServer mockHttpServer; + StreamController requestController; + Testbed testbed; + MockHttpRequest request; + MockHttpResponse response; + MockHttpHeaders headers; + Completer closeCompleter; + WebAssetServer webAssetServer; + MockPlatform windows; + MockPlatform linux; + + setUp(() { + windows = MockPlatform(); + linux = MockPlatform(); + when(windows.environment).thenReturn(const {}); + when(windows.isWindows).thenReturn(true); + when(linux.isWindows).thenReturn(false); + when(linux.environment).thenReturn(const {}); + testbed = Testbed(setup: () { + mockHttpServer = MockHttpServer(); + requestController = StreamController.broadcast(); + request = MockHttpRequest(); + response = MockHttpResponse(); + headers = MockHttpHeaders(); + closeCompleter = Completer(); + when(mockHttpServer.listen(any, onError: anyNamed('onError'))).thenAnswer((Invocation invocation) { + final Function callback = invocation.positionalArguments.first; + return requestController.stream.listen(callback); + }); + when(request.response).thenReturn(response); + when(response.headers).thenReturn(headers); + when(response.close()).thenAnswer((Invocation invocation) async { + closeCompleter.complete(); + }); + webAssetServer = WebAssetServer(mockHttpServer, onError: (dynamic error, StackTrace stackTrace) { + closeCompleter.completeError(error, stackTrace); + }); + }); + }); + + tearDown(() async { + await webAssetServer.dispose(); + await requestController.close(); + }); + + test('Throws a tool exit if bind fails with a SocketException', () => testbed.run(() async { + expect(WebAssetServer.start('hello', 1234), throwsA(isInstanceOf())); + })); + + test('Can catch exceptions through the onError callback', () => testbed.run(() async { + when(response.close()).thenAnswer((Invocation invocation) { + throw StateError('Something bad'); + }); + webAssetServer.writeFile('/foo.js', 'main() {}'); + + when(request.uri).thenReturn(Uri.parse('http://foobar/foo.js')); + requestController.add(request); + + expect(closeCompleter.future, throwsA(isInstanceOf())); + })); + + test('Handles against malformed manifest', () => testbed.run(() async { + final File source = fs.file('source') + ..writeAsStringSync('main() {}'); + + // Missing ending offset. + final File manifestMissingOffset = fs.file('manifestA') + ..writeAsStringSync(json.encode({'/foo.js': [0]})); + // Non-file URI. + final File manifestNonFileScheme = fs.file('manifestA') + ..writeAsStringSync(json.encode({'/foo.js': [0, 10]})); + + final File manifestOutOfBounds = fs.file('manifest') + ..writeAsStringSync(json.encode({'/foo.js': [0, 100]})); + + expect(webAssetServer.write(source, manifestMissingOffset), isEmpty); + expect(webAssetServer.write(source, manifestNonFileScheme), isEmpty); + expect(webAssetServer.write(source, manifestOutOfBounds), isEmpty); + })); + + test('serves JavaScript files from in memory cache', () => testbed.run(() async { + final File source = fs.file('source') + ..writeAsStringSync('main() {}'); + final File manifest = fs.file('manifest') + ..writeAsStringSync(json.encode({'/foo.js': [0, source.lengthSync()]})); + webAssetServer.write(source, manifest); + + when(request.uri).thenReturn(Uri.parse('http://foobar/foo.js')); + requestController.add(request); + await closeCompleter.future; + + verify(headers.add('Content-Length', source.lengthSync())).called(1); + verify(headers.add('Content-Type', 'application/javascript')).called(1); + verify(response.add(source.readAsBytesSync())).called(1); + })); + + test('serves JavaScript files from in memory cache not from manifest', () => testbed.run(() async { + webAssetServer.writeFile('/foo.js', 'main() {}'); + + when(request.uri).thenReturn(Uri.parse('http://foobar/foo.js')); + requestController.add(request); + await closeCompleter.future; + + verify(headers.add('Content-Length', 9)).called(1); + verify(headers.add('Content-Type', 'application/javascript')).called(1); + verify(response.add(any)).called(1); + })); + + test('handles missing JavaScript files from in memory cache', () => testbed.run(() async { + final File source = fs.file('source') + ..writeAsStringSync('main() {}'); + final File manifest = fs.file('manifest') + ..writeAsStringSync(json.encode({'/foo.js': [0, source.lengthSync()]})); + webAssetServer.write(source, manifest); + + when(request.uri).thenReturn(Uri.parse('http://foobar/bar.js')); + requestController.add(request); + await closeCompleter.future; + + verify(response.statusCode = 404).called(1); + })); + + test('serves Dart files from in filesystem on Windows', () => testbed.run(() async { + final File source = fs.file('foo.dart').absolute + ..createSync(recursive: true) + ..writeAsStringSync('void main() {}'); + + when(request.uri).thenReturn(Uri.parse('http://foobar/C:/foo.dart')); + requestController.add(request); + await closeCompleter.future; + + verify(headers.add('Content-Length', source.lengthSync())).called(1); + verify(response.addStream(any)).called(1); + }, overrides: { + Platform: () => windows, + })); + + test('serves Dart files from in filesystem on Linux/macOS', () => testbed.run(() async { + final File source = fs.file('foo.dart').absolute + ..createSync(recursive: true) + ..writeAsStringSync('void main() {}'); + + when(request.uri).thenReturn(Uri.parse('http://foobar/foo.dart')); + requestController.add(request); + await closeCompleter.future; + + verify(headers.add('Content-Length', source.lengthSync())).called(1); + verify(response.addStream(any)).called(1); + }, overrides: { + Platform: () => linux, + })); + + test('Handles missing Dart files from filesystem', () => testbed.run(() async { + when(request.uri).thenReturn(Uri.parse('http://foobar/foo.dart')); + requestController.add(request); + await closeCompleter.future; + + verify(response.statusCode = 404).called(1); + })); + + test('serves asset files from in filesystem with known mime type', () => testbed.run(() async { + final File source = fs.file(fs.path.join('build', 'flutter_assets', 'foo.png')) + ..createSync(recursive: true) + ..writeAsBytesSync(kTransparentImage); + + when(request.uri).thenReturn(Uri.parse('http://foobar/assets/foo.png')); + requestController.add(request); + await closeCompleter.future; + + verify(headers.add('Content-Length', source.lengthSync())).called(1); + verify(headers.add('Content-Type', 'image/png')).called(1); + verify(response.addStream(any)).called(1); + })); + + test('serves asset files from in filesystem with known mime type on Windows', () => testbed.run(() async { + final File source = fs.file(fs.path.join('build', 'flutter_assets', 'foo.png')) + ..createSync(recursive: true) + ..writeAsBytesSync(kTransparentImage); + + when(request.uri).thenReturn(Uri.parse('http://foobar/assets/foo.png')); + requestController.add(request); + await closeCompleter.future; + + verify(headers.add('Content-Length', source.lengthSync())).called(1); + verify(headers.add('Content-Type', 'image/png')).called(1); + verify(response.addStream(any)).called(1); + }, overrides: { + Platform: () => windows, + })); + + + test('serves asset files files from in filesystem with unknown mime type and length > 12', () => testbed.run(() async { + final File source = fs.file(fs.path.join('build', 'flutter_assets', 'foo')) + ..createSync(recursive: true) + ..writeAsBytesSync(List.filled(100, 0)); + + when(request.uri).thenReturn(Uri.parse('http://foobar/assets/foo')); + requestController.add(request); + await closeCompleter.future; + + verify(headers.add('Content-Length', source.lengthSync())).called(1); + verify(headers.add('Content-Type', 'application/octet-stream')).called(1); + verify(response.addStream(any)).called(1); + })); + + test('serves asset files files from in filesystem with unknown mime type and length < 12', () => testbed.run(() async { + final File source = fs.file(fs.path.join('build', 'flutter_assets', 'foo')) + ..createSync(recursive: true) + ..writeAsBytesSync([1, 2, 3]); + + when(request.uri).thenReturn(Uri.parse('http://foobar/assets/foo')); + requestController.add(request); + await closeCompleter.future; + + verify(headers.add('Content-Length', source.lengthSync())).called(1); + verify(headers.add('Content-Type', 'application/octet-stream')).called(1); + verify(response.addStream(any)).called(1); + })); + + test('handles serving missing asset file', () => testbed.run(() async { + when(request.uri).thenReturn(Uri.parse('http://foobar/assets/foo')); + requestController.add(request); + await closeCompleter.future; + + verify(response.statusCode = HttpStatus.notFound).called(1); + })); + + test('calling dispose closes the http server', () => testbed.run(() async { + await webAssetServer.dispose(); + + verify(mockHttpServer.close()).called(1); + })); +} + +class MockHttpServer extends Mock implements HttpServer {} +class MockHttpRequest extends Mock implements HttpRequest {} +class MockHttpResponse extends Mock implements HttpResponse {} +class MockHttpHeaders extends Mock implements HttpHeaders {} +class MockPlatform extends Mock implements Platform {}