Add benchmark tool. (#3802)

_benchmark/README.md

@@ -0,0 +1,8 @@
+Benchmarks `build_runner` against synthetic codebases applying real generators:
+`built_value`, `freezed`, `json_serializable` or `mockito`.
+
+Example usage:
+
+```
+dart run _benchmark --generator=built_value benchmark
+```

_benchmark/bin/_benchmark.dart

@@ -0,0 +1,33 @@
+// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:_benchmark/commands.dart';
+import 'package:_benchmark/generators.dart';
+import 'package:args/command_runner.dart';
+
+final commandRunner =
+    CommandRunner<void>(
+        'dart run _benchmark',
+        'Benchmarks build_runner performance.',
+      )
+      ..addCommand(BenchmarkCommand())
+      ..addCommand(MeasureCommand())
+      ..addCommand(CreateCommand())
+      ..argParser.addOption(
+        'generator',
+        help: 'Generator to benchmark.',
+        allowed: Generator.values.map((e) => e.packageName).toList(),
+        defaultsTo: Generator.builtValue.packageName,
+      )
+      ..argParser.addOption(
+        'root-directory',
+        help: 'Root directory for generated source and builds.',
+        defaultsTo: '${Directory.systemTemp.path}/build_benchmark',
+      );
+
+Future<void> main(List<String> arguments) async {
+  await commandRunner.run(arguments);
+}

_benchmark/lib/benchmark.dart

@@ -0,0 +1,38 @@
+// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'config.dart';
+
+/// A `build_runner` benchmark.
+abstract interface class Benchmark {
+  void create(RunConfig config);
+}
+
+/// Helpers for creating benchmarks.
+class Benchmarks {
+  static String libraryName(int libraryNumber, {required int benchmarkSize}) {
+    // Start numbering from 1.
+    ++libraryNumber;
+    // Pad with zeros so alphabetic sort gives numerical ordering.
+    final sizeDigits = benchmarkSize.toString().length;
+    return 'lib${libraryNumber.toString().padLeft(sizeDigits, '0')}.dart';
+  }
+
+  static String partName(
+    int libraryNumber, {
+    required int benchmarkSize,
+    String infix = 'g',
+  }) => libraryName(
+    libraryNumber,
+    benchmarkSize: benchmarkSize,
+  ).replaceAll('.dart', '.$infix.dart');
+
+  static String testName(int testNumber, {required int benchmarkSize}) {
+    // Start numbering from 1.
+    ++testNumber;
+    // Pad with zeros so alphabetic sort gives numerical ordering.
+    final sizeDigits = benchmarkSize.toString().length;
+    return 'some_test${testNumber.toString().padLeft(sizeDigits, '0')}.dart';
+  }
+}

_benchmark/lib/benchmarks/built_value_generator_benchmark.dart

@@ -0,0 +1,76 @@
+// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import '../benchmark.dart';
+import '../config.dart';
+
+/// Benchmark with one trivial `built_value` value type per library.
+///
+/// There is one large library cycle due to `app.dart` which depends on
+/// everything and is depended on by everything.
+class BuiltValueGeneratorBenchmark implements Benchmark {
+  const BuiltValueGeneratorBenchmark();
+
+  @override
+  void create(RunConfig config) {
+    final workspace = config.workspace;
+    final size = config.size;
+
+    // TODO(davidmorgan): add a way to pick `build` and generator versions.
+    workspace.write(
+      'pubspec.yaml',
+      source: '''
+name: ${workspace.name}
+publish_to: none
+
+environment:
+  sdk: ^3.6.0
+
+dependencies:
+  built_value: any
+
+dev_dependencies:
+  build_runner: any
+  built_value_generator: any
+''',
+    );
+
+    final appLines = ['// ignore_for_file: unused_import', '// CACHEBUSTER'];
+    for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
+      final libraryName = Benchmarks.libraryName(
+        libraryNumber,
+        benchmarkSize: size,
+      );
+      appLines.add("import '$libraryName';");
+    }
+    workspace.write(
+      'lib/app.dart',
+      source: appLines.map((l) => '$l\n').join(''),
+    );
+
+    for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
+      final libraryName = Benchmarks.libraryName(
+        libraryNumber,
+        benchmarkSize: size,
+      );
+      final partName = Benchmarks.partName(libraryNumber, benchmarkSize: size);
+      workspace.write(
+        'lib/$libraryName',
+        source: '''
+// ignore_for_file: unused_import
+import 'package:built_value/built_value.dart';
+
+import 'app.dart';
+
+part '$partName';
+
+abstract class Value implements Built<Value, ValueBuilder> {
+  Value._();
+  factory Value(void Function(ValueBuilder) updates) = _\$Value;
+}
+''',
+      );
+    }
+  }
+}

_benchmark/lib/benchmarks/freezed_generator_benchmark.dart

@@ -0,0 +1,80 @@
+// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import '../benchmark.dart';
+import '../config.dart';
+
+/// Benchmark with one trivial `freezed` value type per library.
+///
+/// There is one large library cycle due to `app.dart` which depends on
+/// everything and is depended on by everything.
+class FreezedGeneratorBenchmark implements Benchmark {
+  const FreezedGeneratorBenchmark();
+
+  @override
+  void create(RunConfig config) {
+    final workspace = config.workspace;
+    final size = config.size;
+
+    // TODO(davidmorgan): add a way to pick `build` and generator versions.
+    workspace.write(
+      'pubspec.yaml',
+      source: '''
+name: ${workspace.name}
+publish_to: none
+
+environment:
+  sdk: ^3.6.0
+
+dependencies:
+  freezed_annotation: any
+
+dev_dependencies:
+  build_runner: any
+  freezed: any
+''',
+    );
+
+    final appLines = ['// ignore_for_file: unused_import', '// CACHEBUSTER'];
+    for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
+      final libraryName = Benchmarks.libraryName(
+        libraryNumber,
+        benchmarkSize: size,
+      );
+      appLines.add("import '$libraryName';");
+    }
+    workspace.write(
+      'lib/app.dart',
+      source: appLines.map((l) => '$l\n').join(''),
+    );
+
+    for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
+      final libraryName = Benchmarks.libraryName(
+        libraryNumber,
+        benchmarkSize: size,
+      );
+      final partName = Benchmarks.partName(
+        libraryNumber,
+        benchmarkSize: size,
+        infix: 'freezed',
+      );
+      workspace.write(
+        'lib/$libraryName',
+        source: '''
+// ignore_for_file: unused_import
+import 'package:freezed_annotation/freezed_annotation.dart';
+
+import 'app.dart';
+
+part '$partName';
+
+@freezed
+class Value with _\$Value {
+  const factory Value() = _Value;
+}
+''',
+      );
+    }
+  }
+}

_benchmark/lib/benchmarks/json_serializable_generator_benchmark.dart

@@ -0,0 +1,80 @@
+// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import '../benchmark.dart';
+import '../config.dart';
+
+/// Benchmark with one trivial `json_serializable` serializable type per
+/// library.
+///
+/// There is one large library cycle due to `app.dart` which depends on
+/// everything and is depended on by everything.
+class JsonSerializableGeneratorBenchmark implements Benchmark {
+  const JsonSerializableGeneratorBenchmark();
+
+  @override
+  void create(RunConfig config) {
+    final workspace = config.workspace;
+    final size = config.size;
+
+    // TODO(davidmorgan): add a way to pick `build` and generator versions.
+    workspace.write(
+      'pubspec.yaml',
+      source: '''
+name: ${workspace.name}
+publish_to: none
+
+environment:
+  sdk: ^3.6.0
+
+dependencies:
+  json_annotation: any
+
+dev_dependencies:
+  build_runner: any
+  json_serializable: any
+''',
+    );
+
+    final appLines = ['// ignore_for_file: unused_import', '// CACHEBUSTER'];
+    for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
+      final libraryName = Benchmarks.libraryName(
+        libraryNumber,
+        benchmarkSize: size,
+      );
+      appLines.add("import '$libraryName';");
+    }
+    workspace.write(
+      'lib/app.dart',
+      source: appLines.map((l) => '$l\n').join(''),
+    );
+
+    for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
+      final libraryName = Benchmarks.libraryName(
+        libraryNumber,
+        benchmarkSize: size,
+      );
+      final partName = Benchmarks.partName(libraryNumber, benchmarkSize: size);
+      workspace.write(
+        'lib/$libraryName',
+        source: '''
+// ignore_for_file: unused_import
+import 'package:json_annotation/json_annotation.dart';
+
+import 'app.dart';
+
+part '$partName';
+
+@JsonSerializable()
+class Value {
+  Value();
+  factory Value.fromJson(Map<String, dynamic> json) =>
+      _\$ValueFromJson(json);
+  Map<String, dynamic> toJson() => _\$ValueToJson(this);
+}
+''',
+      );
+    }
+  }
+}