diff --git a/packages/flutter_tools/lib/src/commands/debug_adapter.dart b/packages/flutter_tools/lib/src/commands/debug_adapter.dart index 77b11341640a0c44b2e01d2a8cfdfe42b44e917f..5ec797b67a57fdb408badba639be80f4162e6724 100644 --- a/packages/flutter_tools/lib/src/commands/debug_adapter.dart +++ b/packages/flutter_tools/lib/src/commands/debug_adapter.dart @@ -28,6 +28,13 @@ class DebugAdapterCommand extends FlutterCommand { DebugAdapterCommand({ bool verboseHelp = false}) : hidden = !verboseHelp { usesIpv6Flag(verboseHelp: verboseHelp); addDdsOptions(verboseHelp: verboseHelp); + argParser + .addFlag( + 'test', + defaultsTo: false, + help: 'Whether to use the "flutter test" debug adapter to run tests' + ' and emit custom events for test progress/results.', + ); } @override @@ -54,6 +61,7 @@ class DebugAdapterCommand extends FlutterCommand { platform: globals.platform, ipv6: ipv6, enableDds: enableDds, + test: boolArg('test') ?? false, ); await server.channel.closed; diff --git a/packages/flutter_tools/lib/src/debug_adapters/README.md b/packages/flutter_tools/lib/src/debug_adapters/README.md index 0a1190a5e63abb4fc5dcbbb10b59690d1c5b6c74..82ae0f77c20711ffd9224ecc1ce9a2fc86ba9c59 100644 --- a/packages/flutter_tools/lib/src/debug_adapters/README.md +++ b/packages/flutter_tools/lib/src/debug_adapters/README.md @@ -4,7 +4,14 @@ This document is Flutter-specific. For information on the standard Dart DAP impl Flutter includes support for debugging using [the Debug Adapter Protocol](https://microsoft.github.io/debug-adapter-protocol/) as an alternative to using the [VM Service](https://github.com/dart-lang/sdk/blob/master/runtime/vm/service/service.md) directly, simplying the integration for new editors. -The debug adapter is started with the `flutter debug-adapter` command and is intended to be consumed by DAP-compliant tools such as Flutter-specific extensions for editors, or configured by users whose editors include generic configurable DAP clients. +The debug adapters are started with the `flutter debug-adapter` command and are intended to be consumed by DAP-compliant tools such as Flutter-specific extensions for editors, or configured by users whose editors include generic configurable DAP clients. + +Two adapters are available: + +- `flutter debug_adapter` +- `flutter debug_adapter --test` + +The standard adapter will run applications using `flutter run` while the `--test` adapter will cause scripts to be run using `flutter test` and will emit custom `dart.testNotification` events (described in the [Dart DAP documentation](https://github.com/dart-lang/sdk/blob/main/pkg/dds/tool/dap/README.md#darttestnotification)). Because in the DAP protocol the client speaks first, running this command from the terminal will result in no output (nor will the process terminate). This is expected behaviour. diff --git a/packages/flutter_tools/lib/src/debug_adapters/flutter_adapter.dart b/packages/flutter_tools/lib/src/debug_adapters/flutter_adapter.dart index d6a7fb16eaf4154c60cbe1fb8bf7f4cfb680c6bb..27b1615249849fa88c18b60b4ab6d78c7de768a7 100644 --- a/packages/flutter_tools/lib/src/debug_adapters/flutter_adapter.dart +++ b/packages/flutter_tools/lib/src/debug_adapters/flutter_adapter.dart @@ -164,9 +164,7 @@ class FlutterDebugAdapter extends DartDebugAdapter toolArgs = [ 'run', '--machine', - if (debug) ...[ - '--start-paused', - ], + if (debug) '--start-paused', ]; final List processArgs = [ ...toolArgs, diff --git a/packages/flutter_tools/lib/src/debug_adapters/flutter_test_adapter.dart b/packages/flutter_tools/lib/src/debug_adapters/flutter_test_adapter.dart new file mode 100644 index 0000000000000000000000000000000000000000..215195473392d85876a9cd2448844c9e073fd66a --- /dev/null +++ b/packages/flutter_tools/lib/src/debug_adapters/flutter_test_adapter.dart @@ -0,0 +1,229 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:dds/dap.dart' hide PidTracker, PackageConfigUtils; +import 'package:vm_service/vm_service.dart' as vm; + +import '../base/file_system.dart'; +import '../base/io.dart'; +import '../base/platform.dart'; +import '../cache.dart'; +import '../convert.dart'; +import 'flutter_adapter_args.dart'; +import 'mixins.dart'; + +/// A DAP Debug Adapter for running and debugging Flutter tests. +class FlutterTestDebugAdapter extends DartDebugAdapter + with PidTracker, PackageConfigUtils, TestAdapter { + FlutterTestDebugAdapter( + ByteStreamServerChannel channel, { + required this.fileSystem, + required this.platform, + bool ipv6 = false, + bool enableDds = true, + bool enableAuthCodes = true, + Logger? logger, + }) : super( + channel, + ipv6: ipv6, + enableDds: enableDds, + enableAuthCodes: enableAuthCodes, + logger: logger, + ); + + @override + FileSystem fileSystem; + Platform platform; + Process? _process; + + @override + final FlutterLaunchRequestArguments Function(Map obj) + parseLaunchArgs = FlutterLaunchRequestArguments.fromJson; + + @override + final FlutterAttachRequestArguments Function(Map obj) + parseAttachArgs = FlutterAttachRequestArguments.fromJson; + + /// Whether the VM Service closing should be used as a signal to terminate the debug session. + /// + /// Since we do not support attaching for tests, this is always false. + @override + bool get terminateOnVmServiceClose => false; + + /// Called by [attachRequest] to request that we actually connect to the app to be debugged. + @override + Future attachImpl() async { + sendOutput('console', '\nAttach is not currently supported'); + handleSessionTerminate(); + } + + @override + Future debuggerConnected(vm.VM vmInfo) async { + // Capture the PID from the VM Service so that we can terminate it when + // cleaning up. Terminating the process might not be enough as it could be + // just a shell script (e.g. pub on Windows) and may not pass the + // signal on correctly. + // See: https://github.com/Dart-Code/Dart-Code/issues/907 + final int? pid = vmInfo.pid; + if (pid != null) { + pidsToTerminate.add(pid); + } + } + + /// Called by [disconnectRequest] to request that we forcefully shut down the app being run (or in the case of an attach, disconnect). + /// + /// Client IDEs/editors should send a terminateRequest before a + /// disconnectRequest to allow a graceful shutdown. This method must terminate + /// quickly and therefore may leave orphaned processes. + @override + Future disconnectImpl() async { + terminatePids(ProcessSignal.sigkill); + } + + /// Called by [launchRequest] to request that we actually start the tests to be run/debugged. + /// + /// For debugging, this should start paused, connect to the VM Service, set + /// breakpoints, and resume. + @override + Future launchImpl() async { + final FlutterLaunchRequestArguments args = this.args as FlutterLaunchRequestArguments; + final String flutterToolPath = fileSystem.path.join(Cache.flutterRoot!, 'bin', platform.isWindows ? 'flutter.bat' : 'flutter'); + + final bool debug = !(args.noDebug ?? false); + final String? program = args.program; + + final List toolArgs = [ + 'test', + '--machine', + if (debug) '--start-paused', + ]; + final List processArgs = [ + ...toolArgs, + ...?args.toolArgs, + if (program != null) program, + ...?args.args, + ]; + + // Find the package_config file for this script. This is used by the + // debugger to map package: URIs to file paths to check whether they're in + // the editors workspace (args.cwd/args.additionalProjectPaths) so they can + // be correctly classes as "my code", "sdk" or "external packages". + // TODO(dantup): Remove this once https://github.com/dart-lang/sdk/issues/45530 + // is done as it will not be necessary. + final String? possibleRoot = program == null + ? args.cwd + : fileSystem.path.isAbsolute(program) + ? fileSystem.path.dirname(program) + : fileSystem.path.dirname( + fileSystem.path.normalize(fileSystem.path.join(args.cwd ?? '', args.program))); + if (possibleRoot != null) { + final File? packageConfig = findPackageConfigFile(possibleRoot); + if (packageConfig != null) { + usePackageConfigFile(packageConfig); + } + } + + logger?.call('Spawning $flutterToolPath with $processArgs in ${args.cwd}'); + final Process process = await Process.start( + flutterToolPath, + processArgs, + workingDirectory: args.cwd, + ); + _process = process; + pidsToTerminate.add(process.pid); + + process.stdout.transform(ByteToLineTransformer()).listen(_handleStdout); + process.stderr.listen(_handleStderr); + unawaited(process.exitCode.then(_handleExitCode)); + + // Delay responding until the debugger is connected. + if (debug) { + await debuggerInitialized; + } + } + + /// Called by [terminateRequest] to request that we gracefully shut down the app being run (or in the case of an attach, disconnect). + @override + Future terminateImpl() async { + terminatePids(ProcessSignal.sigterm); + await _process?.exitCode; + } + + /// Handles the Flutter process exiting, terminating the debug session if it has not already begun terminating. + void _handleExitCode(int code) { + final String codeSuffix = code == 0 ? '' : ' ($code)'; + logger?.call('Process exited ($code)'); + handleSessionTerminate(codeSuffix); + } + + /// Handles incoming JSON events from `flutter test --machine`. + bool _handleJsonEvent(String event, Map? params) { + params ??= {}; + switch (event) { + case 'test.startedProcess': + _handleTestStartedProcess(params); + return true; + } + + return false; + } + + void _handleStderr(List data) { + logger?.call('stderr: $data'); + sendOutput('stderr', utf8.decode(data)); + } + + /// Handles stdout from the `flutter test --machine` process, decoding the JSON and calling the appropriate handlers. + void _handleStdout(String data) { + // Output to stdout from `flutter test --machine` is either: + // 1. JSON output from flutter_tools (eg. "test.startedProcess") which is + // wrapped in [] brackets and has an event/params. + // 2. JSON output from package:test (not wrapped in brackets). + // 3. Non-JSON output (user messages, or flutter_tools printing things like + // call stacks/error information). + logger?.call('stdout: $data'); + + Object? jsonData; + try { + jsonData = jsonDecode(data); + } on FormatException { + // If the output wasn't valid JSON, it was standard stdout that should + // be passed through to the user. + sendOutput('stdout', data); + return; + } + + // Check for valid flutter_tools JSON output (1) first. + final Map? flutterPayload = jsonData is List && + jsonData.length == 1 && + jsonData.first is Map + ? jsonData.first as Map + : null; + final Object? event = flutterPayload?['event']; + final Object? params = flutterPayload?['params']; + + if (event is String && params is Map?) { + _handleJsonEvent(event, params); + } else if (jsonData != null) { + // Handle package:test output (2). + sendTestEvents(jsonData); + } else { + // Other output should just be passed straight through. + sendOutput('stdout', data); + } + } + + /// Handles the test.processStarted event from Flutter that provides the VM Service URL. + void _handleTestStartedProcess(Map params) { + final String? vmServiceUriString = params['observatoryUri'] as String?; + // For no-debug mode, this event is still sent, but has a null URI. + if (vmServiceUriString == null) { + return; + } + final Uri vmServiceUri = Uri.parse(vmServiceUriString); + connectDebugger(vmServiceUri, resumeIfStarting: true); + } +} diff --git a/packages/flutter_tools/lib/src/debug_adapters/server.dart b/packages/flutter_tools/lib/src/debug_adapters/server.dart index 9528c05a51d5934abfb76639b29163cd5f162adb..1a149d2c7372dd4d5f9754950e035a4902a203f8 100644 --- a/packages/flutter_tools/lib/src/debug_adapters/server.dart +++ b/packages/flutter_tools/lib/src/debug_adapters/server.dart @@ -10,6 +10,7 @@ import '../base/file_system.dart'; import '../base/platform.dart'; import '../debug_adapters/flutter_adapter.dart'; import '../debug_adapters/flutter_adapter_args.dart'; +import 'flutter_test_adapter.dart'; /// A DAP server that communicates over a [ByteStreamServerChannel], usually constructed from the processes stdin/stdout streams. /// @@ -27,15 +28,23 @@ class DapServer { this.ipv6 = false, this.enableDds = true, this.enableAuthCodes = true, + bool test = false, this.logger, }) : channel = ByteStreamServerChannel(_input, _output, logger) { - adapter = FlutterDebugAdapter(channel, - fileSystem: fileSystem, - platform: platform, - ipv6: ipv6, - enableDds: enableDds, - enableAuthCodes: enableAuthCodes, - logger: logger); + adapter = test + ? FlutterTestDebugAdapter(channel, + fileSystem: fileSystem, + platform: platform, + ipv6: ipv6, + enableDds: enableDds, + enableAuthCodes: enableAuthCodes, + logger: logger) + : FlutterDebugAdapter(channel, + fileSystem: fileSystem, + platform: platform, + enableDds: enableDds, + enableAuthCodes: enableAuthCodes, + logger: logger); } final ByteStreamServerChannel channel; diff --git a/packages/flutter_tools/test/integration.shard/debug_adapter/flutter_adapter_test.dart b/packages/flutter_tools/test/integration.shard/debug_adapter/flutter_adapter_test.dart index 5e10d5c70732ab83093429408b472d97e70e4f4c..f7d76140fd7fd5491ed33af4a56e160e1b752c3c 100644 --- a/packages/flutter_tools/test/integration.shard/debug_adapter/flutter_adapter_test.dart +++ b/packages/flutter_tools/test/integration.shard/debug_adapter/flutter_adapter_test.dart @@ -27,7 +27,7 @@ void main() { }); setUp(() async { - tempDir = createResolvedTempDirectorySync('debug_adapter_test.'); + tempDir = createResolvedTempDirectorySync('flutter_adapter_test.'); dap = await DapTestSession.setUp(); }); diff --git a/packages/flutter_tools/test/integration.shard/debug_adapter/test_adapter_test.dart b/packages/flutter_tools/test/integration.shard/debug_adapter/test_adapter_test.dart new file mode 100644 index 0000000000000000000000000000000000000000..7fd6a4d302a2237c67b12a9b01316e50f5fae963 --- /dev/null +++ b/packages/flutter_tools/test/integration.shard/debug_adapter/test_adapter_test.dart @@ -0,0 +1,166 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// @dart = 2.8 + +import 'package:dds/src/dap/protocol_generated.dart'; +import 'package:file/file.dart'; +import 'package:flutter_tools/src/cache.dart'; + +import '../../src/common.dart'; +import '../test_data/tests_project.dart'; +import '../test_utils.dart'; +import 'test_client.dart'; +import 'test_support.dart'; + +void main() { + Directory tempDir; + /*late*/ DapTestSession dap; + + setUpAll(() { + Cache.flutterRoot = getFlutterRoot(); + }); + + setUp(() async { + tempDir = createResolvedTempDirectorySync('flutter_test_adapter_test.'); + dap = await DapTestSession.setUp(additionalArgs: ['--test']); + }); + + tearDown(() async { + await dap.tearDown(); + tryToDelete(tempDir); + }); + + test('can run in debug mode', () async { + final DapTestClient client = dap.client; + final TestsProject project = TestsProject(); + await project.setUpIn(tempDir); + + // Collect output and test events while running the script. + final TestEvents outputEvents = await client.collectTestOutput( + launch: () => client.launch( + program: project.testFilePath, + cwd: project.dir.path, + ), + ); + + // Check the printed output shows that the run finished, and it's exit + // code (which is 1 due to the failing test). + final String output = outputEvents.output.map((OutputEventBody e) => e.output).join(); + expectLines( + output, + [ + startsWith('Connecting to VM Service at'), + ..._testsProjectExpectedOutput + ], + allowExtras: true, // Allow for printed call stack etc. + ); + + _expectStandardTestsProjectResults(outputEvents); + }); + + test('can run in noDebug mode', () async { + final DapTestClient client = dap.client; + final TestsProject project = TestsProject(); + await project.setUpIn(tempDir); + + // Collect output and test events while running the script. + final TestEvents outputEvents = await client.collectTestOutput( + launch: () => client.launch( + program: project.testFilePath, + noDebug: true, + cwd: project.dir.path, + ), + ); + + // Check the printed output shows that the run finished, and it's exit + // code (which is 1 due to the failing test). + final String output = outputEvents.output.map((OutputEventBody e) => e.output).join(); + expectLines( + output, + _testsProjectExpectedOutput, + allowExtras: true, // Allow for printed call stack etc. + ); + + _expectStandardTestsProjectResults(outputEvents); + }); + + test('can run a single test', () async { + final DapTestClient client = dap.client; + final TestsProject project = TestsProject(); + await project.setUpIn(tempDir); + + // Collect output and test events while running the script. + final TestEvents outputEvents = await client.collectTestOutput( + launch: () => client.launch( + program: project.testFilePath, + noDebug: true, + cwd: project.dir.path, + // It's up to the calling IDE to pass the correct args for 'dart test' + // if it wants to run a subset of tests. + args: [ + '--plain-name', + 'can pass', + ], + ), + ); + + final List testsNames = outputEvents.testNotifications + .where((Map/*?*/ e) => e['type'] == 'testStart') + .map((Map/*?*/ e) => (e['test'] as Map)['name']) + .toList(); + + expect(testsNames, contains('Flutter tests can pass')); + expect(testsNames, isNot(contains('Flutter tests can fail'))); + }); +} + +/// Matchers for the expected console output of [TestsProject]. +final List _testsProjectExpectedOutput = [ + // First test + '✓ Flutter tests can pass', + // Second test + '══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞════════════════════════════════════════════════════', + 'The following TestFailure object was thrown running a test:', + ' Expected: false', + ' Actual: ', + '', + 'The test description was: can fail', + '', + '✖ Flutter tests can fail', + // Exit + '', + 'Exited (1).', +]; + +/// A helper that verifies a full set of expected test results for the +/// [TestsProject] script. +void _expectStandardTestsProjectResults(TestEvents events) { + // Check we recieved all expected test events passed through from + // package:test. + final List eventNames = + events.testNotifications.map((Map e) => e['type']).toList(); + + // start/done should always be first/last. + expect(eventNames.first, equals('start')); + expect(eventNames.last, equals('done')); + + // allSuites should have occurred after start. + expect( + eventNames, + containsAllInOrder(['start', 'allSuites']), + ); + + // Expect two tests, with the failing one emitting an error. + expect( + eventNames, + containsAllInOrder([ + 'testStart', + 'testDone', + 'testStart', + 'error', + 'testDone', + ]), + ); +} diff --git a/packages/flutter_tools/test/integration.shard/debug_adapter/test_client.dart b/packages/flutter_tools/test/integration.shard/debug_adapter/test_client.dart index 6caa3b2d57d9181ea3ee893780b229c6d58cd0bd..2359787e73ce5c822fa82f142484cee0eb8676c9 100644 --- a/packages/flutter_tools/test/integration.shard/debug_adapter/test_client.dart +++ b/packages/flutter_tools/test/integration.shard/debug_adapter/test_client.dart @@ -74,6 +74,12 @@ class DapTestClient { return _eventController.stream.where((Event e) => e.event == event); } + /// Returns a stream of 'dart.testNotification' custom events from the + /// package:test JSON reporter. + Stream> get testNotificationEvents => + events('dart.testNotification') + .map((Event e) => e.body! as Map); + /// Sends a custom request to the debug adapter to trigger a Hot Reload. Future hotReload() { return custom('hotReload'); @@ -220,6 +226,17 @@ class DapTestClient { } } +/// Useful events produced by the debug adapter during a debug session. +class TestEvents { + TestEvents({ + required this.output, + required this.testNotifications, + }); + + final List output; + final List> testNotifications; +} + class _OutgoingRequest { _OutgoingRequest(this.completer, this.name, this.allowFailure); @@ -273,4 +290,39 @@ extension DapTestClientExtension on DapTestClient { ? output.skipWhile((OutputEventBody output) => output.output.startsWith('Running "flutter pub get"')).toList() : output; } + + /// Collects all output and test events until the program terminates. + /// + /// These results include all events in the order they are recieved, including + /// console, stdout, stderr and test notifications from the test JSON reporter. + /// + /// Only one of [start] or [launch] may be provided. Use [start] to customise + /// the whole start of the session (including initialise) or [launch] to only + /// customise the [launchRequest]. + Future collectTestOutput({ + String? program, + String? cwd, + Future Function()? start, + Future Function()? launch, + }) async { + assert( + start == null || launch == null, + 'Only one of "start" or "launch" may be provided', + ); + + final Future> outputEventsFuture = outputEvents.toList(); + final Future>> testNotificationEventsFuture = testNotificationEvents.toList(); + + if (start != null) { + await start(); + } else { + await this.start(program: program, cwd: cwd, launch: launch); + } + + return TestEvents( + output: await outputEventsFuture, + testNotifications: await testNotificationEventsFuture, + ); + } + } diff --git a/packages/flutter_tools/test/integration.shard/debug_adapter/test_support.dart b/packages/flutter_tools/test/integration.shard/debug_adapter/test_support.dart index 5aafa9497f58288e3f5afb6fd1975765c6fe4dc2..399bb46ef0fc2120aef85f8095d348520081e153 100644 --- a/packages/flutter_tools/test/integration.shard/debug_adapter/test_support.dart +++ b/packages/flutter_tools/test/integration.shard/debug_adapter/test_support.dart @@ -29,11 +29,22 @@ final bool verboseLogging = Platform.environment['DAP_TEST_VERBOSE'] == 'true'; /// Expects the lines in [actual] to match the relevant matcher in [expected], /// ignoring differences in line endings and trailing whitespace. -void expectLines(String actual, List expected) { - expect( - actual.replaceAll('\r\n', '\n').trim().split('\n'), - equals(expected), - ); +void expectLines( + String actual, + List expected, { + bool allowExtras = false, +}) { + if (allowExtras) { + expect( + actual.replaceAll('\r\n', '\n').trim().split('\n'), + containsAllInOrder(expected), + ); + } else { + expect( + actual.replaceAll('\r\n', '\n').trim().split('\n'), + equals(expected), + ); + } } /// A helper class containing the DAP server/client for DAP integration tests. diff --git a/packages/flutter_tools/test/integration.shard/test_data/tests_project.dart b/packages/flutter_tools/test/integration.shard/test_data/tests_project.dart index 15b4c0c5a3b3de890061ff95fa4112f4dd8453ee..c29c417b470198616d655e66f5bf8474735f4a05 100644 --- a/packages/flutter_tools/test/integration.shard/test_data/tests_project.dart +++ b/packages/flutter_tools/test/integration.shard/test_data/tests_project.dart @@ -34,8 +34,13 @@ class TestsProject extends Project { import 'package:flutter_test/flutter_test.dart'; void main() { - testWidgets('Hello world test', (WidgetTester tester) async { - expect(true, isTrue); // BREAKPOINT + group('Flutter tests', () { + testWidgets('can pass', (WidgetTester tester) async { + expect(true, isTrue); // BREAKPOINT + }); + testWidgets('can fail', (WidgetTester tester) async { + expect(true, isFalse); + }); }); } ''';