diff --git a/packages/fleather/example/integration_test/editing_test.dart b/packages/fleather/example/integration_test/editing_test.dart index b8de03fd..0eb0847a 100644 --- a/packages/fleather/example/integration_test/editing_test.dart +++ b/packages/fleather/example/integration_test/editing_test.dart @@ -26,13 +26,13 @@ void main() { await tester.pump(); await tester.ime.typeText(iputText, finder: find.byType(RawEditor)); }, - reportKey: 'editing_timeline', + reportKey: 'timeline', ); }); } final iputText = - 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.'; + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'; final markdown = ''' # Fleather diff --git a/packages/fleather/example/integration_test/scrolling_test.dart b/packages/fleather/example/integration_test/scrolling_test.dart index 00a50ad3..a92abdcd 100644 --- a/packages/fleather/example/integration_test/scrolling_test.dart +++ b/packages/fleather/example/integration_test/scrolling_test.dart @@ -22,18 +22,16 @@ void main() { await binding.traceAction( () async { - for (var i = 0; i < 10; i++) { - while (scrollController.position.extentAfter != 0) { - await tester.drag(scrollableFinder, const Offset(0, -200)); - await tester.pump(); - } - while (scrollController.position.extentBefore != 0) { - await tester.drag(scrollableFinder, const Offset(0, 200)); - await tester.pump(); - } + while (scrollController.position.extentAfter != 0) { + await tester.drag(scrollableFinder, const Offset(0, -500)); + await tester.pump(); + } + while (scrollController.position.extentBefore != 0) { + await tester.drag(scrollableFinder, const Offset(0, 500)); + await tester.pump(); } }, - reportKey: 'scrolling_timeline', + reportKey: 'timeline', ); }); } diff --git a/packages/fleather/example/test_driver/performance_driver.dart b/packages/fleather/example/test_driver/performance_driver.dart index bc879059..aa38113c 100644 --- a/packages/fleather/example/test_driver/performance_driver.dart +++ b/packages/fleather/example/test_driver/performance_driver.dart @@ -1,25 +1,19 @@ +import 'dart:io'; + import 'package:flutter_driver/flutter_driver.dart' as driver; import 'package:integration_test/integration_test_driver.dart'; Future main() { + final outputName = Platform.environment['FLEATHER_PERF_TEST_OUTPUT_NAME']!; return integrationDriver( responseDataCallback: (data) async { if (data != null) { - await writeTimeline(data, 'scrolling_timeline', 'scrolling'); - await writeTimeline(data, 'editing_timeline', 'editing'); + final timeline = + driver.Timeline.fromJson(data['timeline'] as Map); + final summary = driver.TimelineSummary.summarize(timeline); + await summary.writeTimelineToFile(outputName, + pretty: true, includeSummary: true); } }, ); } - -Future writeTimeline( - Map data, String key, String name) async { - if (!data.containsKey(key)) return; - - final timeline = driver.Timeline.fromJson(data[key] as Map); - final summary = driver.TimelineSummary.summarize(timeline); - await summary.writeTimelineToFile(name, - destinationDirectory: 'build/performance_timelines', - pretty: true, - includeSummary: true); -} diff --git a/packages/fleather/example/test_utils/analyze_performance_result.dart b/packages/fleather/example/test_utils/analyze_performance.dart similarity index 60% rename from packages/fleather/example/test_utils/analyze_performance_result.dart rename to packages/fleather/example/test_utils/analyze_performance.dart index c8a0faf3..6cc4d2fa 100644 --- a/packages/fleather/example/test_utils/analyze_performance_result.dart +++ b/packages/fleather/example/test_utils/analyze_performance.dart @@ -26,22 +26,16 @@ const _criterias = [ Criteria('average_cpu_usage', 'Average CPU Usage'), ]; -const _performanceTimelinesPath = 'build/performance_timelines'; -const _referencePerformanceTimelinesPath = - 'build/reference_performance_timelines'; - -void main() { +void main(List args) { final outputBuffer = StringBuffer(); bool hasRegression = false; - final performanceTimelinesDir = Directory(_performanceTimelinesPath); - for (final fileEntity in performanceTimelinesDir.listSync()) { - if (!fileEntity.path.contains('timeline_summary.json')) continue; - final fileName = fileEntity.path.split('/').last; - final result = _analyze(fileName); - hasRegression = hasRegression || result.$2; - outputBuffer.writeln(result.$1); - } + final reference = args[0]; + final target = args[1]; + + final result = _analyze(reference, target); + hasRegression = hasRegression || result.$2; + outputBuffer.writeln(result.$1); if (hasRegression) { outputBuffer.write(buildErrorMessage('Performance tests found regression')); @@ -54,15 +48,11 @@ void main() { exit(hasRegression ? 1 : 0); } -(String, bool) _analyze(String fileName) { - final Map targetSummary = jsonDecode( - File('$_performanceTimelinesPath/$fileName').readAsStringSync()); - Map? referenceSummary; - final referenceSummaryFile = - File('$_referencePerformanceTimelinesPath/$fileName'); - if (referenceSummaryFile.existsSync()) { - referenceSummary = jsonDecode(referenceSummaryFile.readAsStringSync()); - } +(String, bool) _analyze(String reference, String target) { + final Map referenceSummary = + jsonDecode(File(reference).readAsStringSync()); + final Map targetSummary = + jsonDecode(File(target).readAsStringSync()); bool testHasRegression = false; final outputBuffer = StringBuffer(); @@ -71,28 +61,19 @@ void main() { if (!targetSummary.containsKey(criteria.key)) continue; bool criteriaHasRegression = false; final double targetValue = targetSummary[criteria.key]; - final double? referenceValue = referenceSummary?[criteria.key]; - if (referenceValue != null) { - criteriaHasRegression = - criteria.isRegression(targetValue, referenceValue); - } + final double referenceValue = referenceSummary[criteria.key]; + criteriaHasRegression = criteria.isRegression(targetValue, referenceValue); outputBuffer.write('${criteria.title}: Target: '); - if (referenceValue != null && - criteria.isWorse(targetValue, referenceValue)) { + if (criteria.isWorse(targetValue, referenceValue)) { outputBuffer.write(buildErrorMessage(targetValue.toStringAsFixed(2))); } else { outputBuffer.write(targetValue.toStringAsFixed(2)); } - if (referenceValue != null) { - outputBuffer.writeln(' Reference: ${referenceValue.toStringAsFixed(2)}'); - } else { - outputBuffer.writeln(); - } + outputBuffer.writeln(' Reference: ${referenceValue.toStringAsFixed(2)}'); testHasRegression = testHasRegression || criteriaHasRegression; } - outputBuffer - .write('Performance tests for ${fileName.split('.').first} found '); + outputBuffer.write('Performance tests found '); if (testHasRegression) { outputBuffer.writeln(buildErrorMessage('regression')); } else { diff --git a/packages/fleather/example/test_utils/run_and_analyze_performance_tests.dart b/packages/fleather/example/test_utils/run_and_analyze_performance_tests.dart new file mode 100644 index 00000000..4cfac316 --- /dev/null +++ b/packages/fleather/example/test_utils/run_and_analyze_performance_tests.dart @@ -0,0 +1,93 @@ +import 'dart:io'; + +import 'package:integration_test/integration_test_driver.dart'; + +/// Runs and analyzes performance tests. +/// +/// It's an internal tests for developers and maintainers, and is not safe +/// since you might end up losing your changes. +/// Use with care and commit changes in current branch and master before. +void main() { + final branchName = + Process.runSync('git', ['rev-parse', '--abbrev-ref', 'HEAD']) + .stdout + .toString() + .trim(); + + warning('Running tests for $branchName'); + Process.runSync( + 'flutter', + [ + 'drive', + '-d', + 'macos', + '--driver=test_driver/performance_driver.dart', + '--target=integration_test/scrolling_test.dart', + '--profile', + ], + environment: {'FLEATHER_PERF_TEST_OUTPUT_NAME': 'target_scrolling'}, + ); + Process.runSync( + 'flutter', + [ + 'drive', + '-d', + 'macos', + '--driver=test_driver/performance_driver.dart', + '--target=integration_test/editing_test.dart', + '--profile', + ], + environment: {'FLEATHER_PERF_TEST_OUTPUT_NAME': 'target_editing'}, + ); + + Process.runSync('git', ['stash', '--include-untracked']); + Process.runSync('git', ['fetch', 'origin/master']); + Process.runSync('git', ['checkout', 'origin/master']); + + warning('Running tests for origin/master'); + Process.runSync( + 'flutter', + [ + 'drive', + '-d', + 'macos', + '--driver=test_driver/performance_driver.dart', + '--target=integration_test/scrolling_test.dart', + '--profile', + ], + environment: {'FLEATHER_PERF_TEST_OUTPUT_NAME': 'ref_scrolling'}, + ); + Process.runSync( + 'flutter', + [ + 'drive', + '-d', + 'macos', + '--driver=test_driver/performance_driver.dart', + '--target=integration_test/editing_test.dart', + '--profile', + ], + environment: {'FLEATHER_PERF_TEST_OUTPUT_NAME': 'ref_editing'}, + ); + + warning('Analyzing tests for scrolling'); + print(Process.runSync('dart', [ + 'run', + 'test_utils/analyze_performance.dart', + '$testOutputsDirectory/ref_scrolling.timeline_summary.json', + '$testOutputsDirectory/target_scrolling.timeline_summary.json', + ]).stdout); + + warning('Analyzing tests for editing'); + print(Process.runSync('dart', [ + 'run', + 'test_utils/analyze_performance.dart', + '$testOutputsDirectory/ref_editing.timeline_summary.json', + '$testOutputsDirectory/target_editing.timeline_summary.json', + ]).stdout); + + Process.runSync('git', ['checkout', branchName]); + Process.runSync('git', ['stash', 'apply']); +} + +void warning(String text) => print('\x1B[33m$text\x1B[0m');