From afd2645d23433c69c1bd216d75b0171e78bd7d9c Mon Sep 17 00:00:00 2001 From: David Morgan Date: Fri, 31 Jan 2025 01:36:41 -0800 Subject: [PATCH] Add benchmark tool. (#3802) --- _benchmark/README.md | 8 ++ _benchmark/bin/_benchmark.dart | 33 +++++ _benchmark/lib/benchmark.dart | 38 ++++++ .../built_value_generator_benchmark.dart | 76 +++++++++++ .../freezed_generator_benchmark.dart | 80 +++++++++++ ...json_serializable_generator_benchmark.dart | 80 +++++++++++ .../mockito_generator_benchmark.dart | 95 +++++++++++++ _benchmark/lib/commands.dart | 114 ++++++++++++++++ _benchmark/lib/config.dart | 44 ++++++ _benchmark/lib/generators.dart | 28 ++++ _benchmark/lib/workspace.dart | 126 ++++++++++++++++++ _benchmark/pubspec.yaml | 10 ++ 12 files changed, 732 insertions(+) create mode 100644 _benchmark/README.md create mode 100644 _benchmark/bin/_benchmark.dart create mode 100644 _benchmark/lib/benchmark.dart create mode 100644 _benchmark/lib/benchmarks/built_value_generator_benchmark.dart create mode 100644 _benchmark/lib/benchmarks/freezed_generator_benchmark.dart create mode 100644 _benchmark/lib/benchmarks/json_serializable_generator_benchmark.dart create mode 100644 _benchmark/lib/benchmarks/mockito_generator_benchmark.dart create mode 100644 _benchmark/lib/commands.dart create mode 100644 _benchmark/lib/config.dart create mode 100644 _benchmark/lib/generators.dart create mode 100644 _benchmark/lib/workspace.dart create mode 100644 _benchmark/pubspec.yaml diff --git a/_benchmark/README.md b/_benchmark/README.md new file mode 100644 index 000000000..41f456d3f --- /dev/null +++ b/_benchmark/README.md @@ -0,0 +1,8 @@ +Benchmarks `build_runner` against synthetic codebases applying real generators: +`built_value`, `freezed`, `json_serializable` or `mockito`. + +Example usage: + +``` +dart run _benchmark --generator=built_value benchmark +``` diff --git a/_benchmark/bin/_benchmark.dart b/_benchmark/bin/_benchmark.dart new file mode 100644 index 000000000..dea1db182 --- /dev/null +++ b/_benchmark/bin/_benchmark.dart @@ -0,0 +1,33 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:_benchmark/commands.dart'; +import 'package:_benchmark/generators.dart'; +import 'package:args/command_runner.dart'; + +final commandRunner = + CommandRunner( + 'dart run _benchmark', + 'Benchmarks build_runner performance.', + ) + ..addCommand(BenchmarkCommand()) + ..addCommand(MeasureCommand()) + ..addCommand(CreateCommand()) + ..argParser.addOption( + 'generator', + help: 'Generator to benchmark.', + allowed: Generator.values.map((e) => e.packageName).toList(), + defaultsTo: Generator.builtValue.packageName, + ) + ..argParser.addOption( + 'root-directory', + help: 'Root directory for generated source and builds.', + defaultsTo: '${Directory.systemTemp.path}/build_benchmark', + ); + +Future main(List arguments) async { + await commandRunner.run(arguments); +} diff --git a/_benchmark/lib/benchmark.dart b/_benchmark/lib/benchmark.dart new file mode 100644 index 000000000..7fdef8062 --- /dev/null +++ b/_benchmark/lib/benchmark.dart @@ -0,0 +1,38 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'config.dart'; + +/// A `build_runner` benchmark. +abstract interface class Benchmark { + void create(RunConfig config); +} + +/// Helpers for creating benchmarks. +class Benchmarks { + static String libraryName(int libraryNumber, {required int benchmarkSize}) { + // Start numbering from 1. + ++libraryNumber; + // Pad with zeros so alphabetic sort gives numerical ordering. + final sizeDigits = benchmarkSize.toString().length; + return 'lib${libraryNumber.toString().padLeft(sizeDigits, '0')}.dart'; + } + + static String partName( + int libraryNumber, { + required int benchmarkSize, + String infix = 'g', + }) => libraryName( + libraryNumber, + benchmarkSize: benchmarkSize, + ).replaceAll('.dart', '.$infix.dart'); + + static String testName(int testNumber, {required int benchmarkSize}) { + // Start numbering from 1. + ++testNumber; + // Pad with zeros so alphabetic sort gives numerical ordering. + final sizeDigits = benchmarkSize.toString().length; + return 'some_test${testNumber.toString().padLeft(sizeDigits, '0')}.dart'; + } +} diff --git a/_benchmark/lib/benchmarks/built_value_generator_benchmark.dart b/_benchmark/lib/benchmarks/built_value_generator_benchmark.dart new file mode 100644 index 000000000..75940a20a --- /dev/null +++ b/_benchmark/lib/benchmarks/built_value_generator_benchmark.dart @@ -0,0 +1,76 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import '../benchmark.dart'; +import '../config.dart'; + +/// Benchmark with one trivial `built_value` value type per library. +/// +/// There is one large library cycle due to `app.dart` which depends on +/// everything and is depended on by everything. +class BuiltValueGeneratorBenchmark implements Benchmark { + const BuiltValueGeneratorBenchmark(); + + @override + void create(RunConfig config) { + final workspace = config.workspace; + final size = config.size; + + // TODO(davidmorgan): add a way to pick `build` and generator versions. + workspace.write( + 'pubspec.yaml', + source: ''' +name: ${workspace.name} +publish_to: none + +environment: + sdk: ^3.6.0 + +dependencies: + built_value: any + +dev_dependencies: + build_runner: any + built_value_generator: any +''', + ); + + final appLines = ['// ignore_for_file: unused_import', '// CACHEBUSTER']; + for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) { + final libraryName = Benchmarks.libraryName( + libraryNumber, + benchmarkSize: size, + ); + appLines.add("import '$libraryName';"); + } + workspace.write( + 'lib/app.dart', + source: appLines.map((l) => '$l\n').join(''), + ); + + for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) { + final libraryName = Benchmarks.libraryName( + libraryNumber, + benchmarkSize: size, + ); + final partName = Benchmarks.partName(libraryNumber, benchmarkSize: size); + workspace.write( + 'lib/$libraryName', + source: ''' +// ignore_for_file: unused_import +import 'package:built_value/built_value.dart'; + +import 'app.dart'; + +part '$partName'; + +abstract class Value implements Built { + Value._(); + factory Value(void Function(ValueBuilder) updates) = _\$Value; +} +''', + ); + } + } +} diff --git a/_benchmark/lib/benchmarks/freezed_generator_benchmark.dart b/_benchmark/lib/benchmarks/freezed_generator_benchmark.dart new file mode 100644 index 000000000..91007ddc2 --- /dev/null +++ b/_benchmark/lib/benchmarks/freezed_generator_benchmark.dart @@ -0,0 +1,80 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import '../benchmark.dart'; +import '../config.dart'; + +/// Benchmark with one trivial `freezed` value type per library. +/// +/// There is one large library cycle due to `app.dart` which depends on +/// everything and is depended on by everything. +class FreezedGeneratorBenchmark implements Benchmark { + const FreezedGeneratorBenchmark(); + + @override + void create(RunConfig config) { + final workspace = config.workspace; + final size = config.size; + + // TODO(davidmorgan): add a way to pick `build` and generator versions. + workspace.write( + 'pubspec.yaml', + source: ''' +name: ${workspace.name} +publish_to: none + +environment: + sdk: ^3.6.0 + +dependencies: + freezed_annotation: any + +dev_dependencies: + build_runner: any + freezed: any +''', + ); + + final appLines = ['// ignore_for_file: unused_import', '// CACHEBUSTER']; + for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) { + final libraryName = Benchmarks.libraryName( + libraryNumber, + benchmarkSize: size, + ); + appLines.add("import '$libraryName';"); + } + workspace.write( + 'lib/app.dart', + source: appLines.map((l) => '$l\n').join(''), + ); + + for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) { + final libraryName = Benchmarks.libraryName( + libraryNumber, + benchmarkSize: size, + ); + final partName = Benchmarks.partName( + libraryNumber, + benchmarkSize: size, + infix: 'freezed', + ); + workspace.write( + 'lib/$libraryName', + source: ''' +// ignore_for_file: unused_import +import 'package:freezed_annotation/freezed_annotation.dart'; + +import 'app.dart'; + +part '$partName'; + +@freezed +class Value with _\$Value { + const factory Value() = _Value; +} +''', + ); + } + } +} diff --git a/_benchmark/lib/benchmarks/json_serializable_generator_benchmark.dart b/_benchmark/lib/benchmarks/json_serializable_generator_benchmark.dart new file mode 100644 index 000000000..cd2cb0d91 --- /dev/null +++ b/_benchmark/lib/benchmarks/json_serializable_generator_benchmark.dart @@ -0,0 +1,80 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import '../benchmark.dart'; +import '../config.dart'; + +/// Benchmark with one trivial `json_serializable` serializable type per +/// library. +/// +/// There is one large library cycle due to `app.dart` which depends on +/// everything and is depended on by everything. +class JsonSerializableGeneratorBenchmark implements Benchmark { + const JsonSerializableGeneratorBenchmark(); + + @override + void create(RunConfig config) { + final workspace = config.workspace; + final size = config.size; + + // TODO(davidmorgan): add a way to pick `build` and generator versions. + workspace.write( + 'pubspec.yaml', + source: ''' +name: ${workspace.name} +publish_to: none + +environment: + sdk: ^3.6.0 + +dependencies: + json_annotation: any + +dev_dependencies: + build_runner: any + json_serializable: any +''', + ); + + final appLines = ['// ignore_for_file: unused_import', '// CACHEBUSTER']; + for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) { + final libraryName = Benchmarks.libraryName( + libraryNumber, + benchmarkSize: size, + ); + appLines.add("import '$libraryName';"); + } + workspace.write( + 'lib/app.dart', + source: appLines.map((l) => '$l\n').join(''), + ); + + for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) { + final libraryName = Benchmarks.libraryName( + libraryNumber, + benchmarkSize: size, + ); + final partName = Benchmarks.partName(libraryNumber, benchmarkSize: size); + workspace.write( + 'lib/$libraryName', + source: ''' +// ignore_for_file: unused_import +import 'package:json_annotation/json_annotation.dart'; + +import 'app.dart'; + +part '$partName'; + +@JsonSerializable() +class Value { + Value(); + factory Value.fromJson(Map json) => + _\$ValueFromJson(json); + Map toJson() => _\$ValueToJson(this); +} +''', + ); + } + } +} diff --git a/_benchmark/lib/benchmarks/mockito_generator_benchmark.dart b/_benchmark/lib/benchmarks/mockito_generator_benchmark.dart new file mode 100644 index 000000000..3223e100f --- /dev/null +++ b/_benchmark/lib/benchmarks/mockito_generator_benchmark.dart @@ -0,0 +1,95 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import '../benchmark.dart'; +import '../config.dart'; + +/// Benchmark with one trivial service class and one test per library. +/// +/// Each test creates a mock of only its corresponding service class, so +/// a test uses exactly one mock. +/// +/// There is one large library cycle due to `app.dart` which depends on +/// everything and is depended on by everything. +class MockitoGeneratorBenchmark implements Benchmark { + const MockitoGeneratorBenchmark(); + + @override + void create(RunConfig config) { + final workspace = config.workspace; + final size = config.size; + + // TODO(davidmorgan): add a way to pick `build` and generator versions. + workspace.write( + 'pubspec.yaml', + source: ''' +name: ${workspace.name} +publish_to: none + +environment: + sdk: ^3.6.0 + +dependencies: + mockito: any + +dev_dependencies: + build_runner: any +''', + ); + + final appLines = ['// ignore_for_file: unused_import', '// CACHEBUSTER']; + for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) { + final libraryName = Benchmarks.libraryName( + libraryNumber, + benchmarkSize: size, + ); + appLines.add("import '$libraryName';"); + } + workspace.write( + 'lib/app.dart', + source: appLines.map((l) => '$l\n').join(''), + ); + + for (var testNumber = 0; testNumber != size; ++testNumber) { + final testLines = [ + '// ignore_for_file: unused import', + '// CACHEBUSTER', + "import 'package:mockito/annotations.dart';", + ]; + final libraryName = Benchmarks.libraryName( + testNumber, + benchmarkSize: size, + ); + testLines.add("import 'package:${workspace.name}/$libraryName';"); + testLines.add('@GenerateNiceMocks(['); + testLines.add('MockSpec(),'); + testLines.add('])'); + testLines.add("import 'some_test.mocks.dart';"); + + final testName = Benchmarks.testName(testNumber, benchmarkSize: size); + workspace.write( + 'test/$testName', + source: testLines.map((l) => '$l\n').join(''), + ); + } + + for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) { + final libraryName = Benchmarks.libraryName( + libraryNumber, + benchmarkSize: size, + ); + workspace.write( + 'lib/$libraryName', + source: ''' +// ignore_for_file: unused_import +import 'app.dart'; + +class Service$libraryNumber { + void doSomething(int value) {} +} +''', + ); + } + } +} diff --git a/_benchmark/lib/commands.dart b/_benchmark/lib/commands.dart new file mode 100644 index 000000000..b55798ac9 --- /dev/null +++ b/_benchmark/lib/commands.dart @@ -0,0 +1,114 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:args/args.dart'; +import 'package:args/command_runner.dart'; + +import 'config.dart'; +import 'workspace.dart'; + +class BenchmarkCommand extends Command { + @override + String get description => 'Runs "create" then "measure".'; + + @override + String get name => 'benchmark'; + + @override + Future run() async { + await CreateCommand()._run(globalResults!); + await MeasureCommand()._run(globalResults!); + } +} + +class CreateCommand extends Command { + @override + String get description => 'Creates codebases to benchmark.'; + + @override + String get name => 'create'; + + @override + Future run() => _run(globalResults!); + + Future _run(ArgResults globalResults) async { + final config = Config.fromArgResults(globalResults!); + + for (final size in config.sizes) { + final paddedSize = size.toString().padLeft(4, '0'); + final workspace = Workspace( + config: config, + name: '${config.generator.packageName}_$paddedSize', + ); + final runConfig = RunConfig( + config: config, + workspace: workspace, + size: size, + paddedSize: paddedSize, + ); + config.generator.benchmark.create(runConfig); + } + } +} + +class MeasureCommand extends Command { + @override + String get description => 'Builds and measures performance.'; + + @override + String get name => 'measure'; + + @override + Future run() => _run(globalResults!); + + Future _run(ArgResults globalResults) async { + // Launch a benchmark at each size in parallel. + final config = Config.fromArgResults(globalResults!); + final pendingResults = {}; + for (final size in config.sizes) { + final paddedSize = size.toString().padLeft(4, '0'); + final workspace = Workspace( + config: config, + name: '${config.generator.packageName}_$paddedSize', + clean: false, + ); + pendingResults[size] = workspace.measure(); + } + + // Wait for them to complete, printing status every second only if it + // changed. + String? previousUpdate; + while (!pendingResults.values.every((r) => r.isFinished)) { + await Future.delayed(const Duration(seconds: 1)); + + final update = StringBuffer('${config.generator.packageName}\n'); + update.write('libraries,clean/ms,no changes/ms,incremental/ms\n'); + for (final size in config.sizes) { + final pendingResult = pendingResults[size]!; + if (pendingResult.isFailure) { + throw StateError(pendingResult.failure!); + } + update.write( + [ + size, + pendingResults[size]!.cleanBuildTime.render, + pendingResults[size]!.noChangesBuildTime.render, + pendingResults[size]!.incrementalBuildTime.render, + ].join(','), + ); + update.write('\n'); + } + + final updateString = update.toString(); + if (updateString != previousUpdate) { + print(updateString); + previousUpdate = updateString; + } + } + } +} + +extension DurationExtension on Duration? { + String get render => this == null ? '---' : this!.inMilliseconds.toString(); +} diff --git a/_benchmark/lib/config.dart b/_benchmark/lib/config.dart new file mode 100644 index 000000000..e3b1952bb --- /dev/null +++ b/_benchmark/lib/config.dart @@ -0,0 +1,44 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:args/args.dart'; + +import 'generators.dart'; +import 'workspace.dart'; + +/// Benchmark tool config. +class Config { + final Generator generator; + final Directory rootDirectory; + final List sizes = const [1, 100, 250, 500, 750, 1000]; + + Config({required this.generator, required this.rootDirectory}); + + factory Config.fromArgResults(ArgResults argResults) => Config( + generator: Generator.values.singleWhere( + (e) => e.packageName == argResults['generator'], + ), + rootDirectory: Directory(argResults['root-directory'] as String), + ); +} + +/// Single benchmark run config. +class RunConfig { + final Config config; + final int size; + + /// [size] as a padded-to-consistent-width `String`. + final String paddedSize; + + final Workspace workspace; + + RunConfig({ + required this.config, + required this.workspace, + required this.size, + required this.paddedSize, + }); +} diff --git a/_benchmark/lib/generators.dart b/_benchmark/lib/generators.dart new file mode 100644 index 000000000..e63f80d0d --- /dev/null +++ b/_benchmark/lib/generators.dart @@ -0,0 +1,28 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'benchmark.dart'; +import 'benchmarks/built_value_generator_benchmark.dart'; +import 'benchmarks/freezed_generator_benchmark.dart'; +import 'benchmarks/json_serializable_generator_benchmark.dart'; +import 'benchmarks/mockito_generator_benchmark.dart'; + +/// A code generator for benchmarking. +enum Generator { + builtValue( + packageName: 'built_value', + benchmark: BuiltValueGeneratorBenchmark(), + ), + freezed(packageName: 'freezed', benchmark: FreezedGeneratorBenchmark()), + jsonSerializable( + packageName: 'json_serializable', + benchmark: JsonSerializableGeneratorBenchmark(), + ), + mockito(packageName: 'mockito', benchmark: MockitoGeneratorBenchmark()); + + final String packageName; + final Benchmark benchmark; + + const Generator({required this.packageName, required this.benchmark}); +} diff --git a/_benchmark/lib/workspace.dart b/_benchmark/lib/workspace.dart new file mode 100644 index 000000000..beea87a85 --- /dev/null +++ b/_benchmark/lib/workspace.dart @@ -0,0 +1,126 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +import 'config.dart'; + +/// A temporary workspace for benchmarking `build_runner`. +class Workspace { + final Config config; + final String name; + + Directory get directory => + Directory.fromUri(config.rootDirectory.uri.resolve(name)); + + Workspace({required this.config, required this.name, bool clean = true}) { + if (clean) { + if (directory.existsSync()) directory.deleteSync(recursive: true); + directory.createSync(recursive: true); + } + } + + /// Writes [source] to [path]. + void write(String path, {required String source}) { + final file = File.fromUri(directory.uri.resolve(path)); + file.parent.createSync(recursive: true); + file.writeAsStringSync(source); + } + + /// Edits the file at [path] to replace `CACHEBUSTER` with a timestamp. + /// + /// Throws `StateError` if the string is not present to replace. + void edit(String path) { + final file = File.fromUri(directory.uri.resolve(path)); + final source = file.readAsStringSync(); + final edited = source.replaceAll( + 'CACHEBUSTER', + 'CACHEBUSTER${DateTime.now().microsecondsSinceEpoch.toString()}', + ); + if (source == edited) { + throw StateError('Edit of "$path" failed: no CACHEBUSTER in source.'); + } + file.writeAsStringSync(edited); + } + + /// Builds and measures performance. + PendingResult measure() { + final result = PendingResult(); + _measure(result); + return result; + } + + Future _measure(PendingResult result) async { + // Clean build. + final stopwatch = Stopwatch()..start(); + var process = await Process.start('dart', [ + 'run', + 'build_runner', + 'build', + '-d', + ], workingDirectory: directory.path); + var exitCode = await process.exitCode; + result.cleanBuildTime = stopwatch.elapsed; + if (exitCode != 0) { + final stdout = await process.stdout.transform(utf8.decoder).join(); + final stderr = await process.stderr.transform(utf8.decoder).join(); + result.failure = 'Initial build failed:\n$stdout\n$stderr'; + return; + } + + // Build with no changes. + stopwatch.reset(); + process = await Process.start('dart', [ + 'run', + 'build_runner', + 'build', + '-d', + ], workingDirectory: directory.path); + exitCode = await process.exitCode; + result.noChangesBuildTime = stopwatch.elapsed; + if (exitCode != 0) { + final stdout = await process.stdout.transform(utf8.decoder).join(); + final stderr = await process.stderr.transform(utf8.decoder).join(); + result.failure = 'No changes build failed:\n$stdout\n$stderr'; + return; + } + + // Incremental build after a change. + edit('lib/app.dart'); + stopwatch.reset(); + process = await Process.start('dart', [ + 'run', + 'build_runner', + 'build', + '-d', + ], workingDirectory: directory.path); + exitCode = await process.exitCode; + result.incrementalBuildTime = stopwatch.elapsed; + if (exitCode != 0) { + final stdout = await process.stdout.transform(utf8.decoder).join(); + final stderr = await process.stderr.transform(utf8.decoder).join(); + result.failure = 'Incremental build failed:\n$stdout\n$stderr'; + return; + } + } +} + +/// Benchmark results. +/// +/// May be partial if the benchmark is still running. +class PendingResult { + Duration? cleanBuildTime; + Duration? noChangesBuildTime; + Duration? incrementalBuildTime; + String? failure; + + bool get isFailure => failure != null; + bool get isSuccess => + !isFailure && + cleanBuildTime != null && + noChangesBuildTime != null && + incrementalBuildTime != null; + bool get isFinished => isSuccess || isFailure; +} diff --git a/_benchmark/pubspec.yaml b/_benchmark/pubspec.yaml new file mode 100644 index 000000000..6aa62b368 --- /dev/null +++ b/_benchmark/pubspec.yaml @@ -0,0 +1,10 @@ +name: _benchmark +publish_to: none +environment: + sdk: ^3.7.0-edge + +dependencies: + args: ^2.6.0 + +dev_dependencies: + dart_flutter_team_lints: ^3.0.0