diff --git a/docs/filters.mdx b/docs/filters.mdx index d3bc8732..7ed788d5 100644 --- a/docs/filters.mdx +++ b/docs/filters.mdx @@ -5,9 +5,21 @@ description: Learn more about all the package filtering flags in Melos. # Filtering Packages -Each Melos command can be used alongside the following global filters: +Each Melos command can be used alongside the following global filters. -## --no-private +The filters are divided into 2 priority groups. The priority determines the +order of their application. The grouping has been introduced with backward +compatibility in mind. It is possible to reverse the order of the priority +groups to achive different filtering result. + +## --reversed-filter-group-priority +Reverses the priority of filtering groups. Useful for applying diff-based +filtering first, allowing the filters from the other group to further narrow +down the packages. + +## 1st Priority Group + +### --no-private Exclude private packages (`publish_to: none`). They are included by default. @@ -15,7 +27,7 @@ Exclude private packages (`publish_to: none`). They are included by default. melos bootstrap --no-private ``` -## --published +### --published Filter packages where the current local package version exists on pub.dev. @@ -26,7 +38,7 @@ melos bootstrap --published Use `--no-published` to filter packages that have not had their current version published yet. -## --scope +### --scope Include only packages with names matching the given glob. This option can be repeated. @@ -36,7 +48,7 @@ repeated. melos exec --scope="*example*" -- flutter build ios ``` -## --ignore +### --ignore Exclude packages with names matching the given glob. This option can be repeated. @@ -46,25 +58,7 @@ repeated. melos exec --ignore="*internal*" -- flutter build ios ``` -## --diff - -Filter packages based on whether there were changes between a commit and the -current HEAD or within a range of commits. - -A range of commits can be specified using the git short hand syntax -`..` and `...`. - -```bash -# Run `flutter build ios` on all packages that are different between current -# branch and the specified commit hash. -melos exec --diff= -- flutter build ios - -# Run `flutter build ios` on all packages that are different between remote -# `main` branch and HEAD. -melos exec --diff=origin/main...HEAD -- flutter build ios -``` - -## --dir-exists +### --dir-exists Include only packages where a specific directory exists inside the package. @@ -73,7 +67,7 @@ Include only packages where a specific directory exists inside the package. melos bootstrap --dir-exists="example" ``` -## --file-exists +### --file-exists Include only packages where a specific file exists in the package. @@ -82,7 +76,7 @@ Include only packages where a specific file exists in the package. melos bootstrap --file-exists="README.md" ``` -## --flutter +### --flutter Filter packages where the package depends on the Flutter SDK. @@ -92,7 +86,7 @@ melos exec --flutter -- flutter test Use `--no-flutter` to filter packages that do not depend on the Flutter SDK. -## --depends-on +### --depends-on Include only packages that depend on specific dependencies. @@ -103,7 +97,32 @@ melos exec --depends-on="flutter" --depends-on="firebase_core" -- flutter test Use `--no-depends-on` to filter packages that do not depend on the given dependencies. -## --include-dependencies +## 2nd Priority Group + +### --diff + +Filter packages based on whether there were changes between a commit and the +current HEAD or within a range of commits. + +A range of commits can be specified using the git short hand syntax +`..` and `...`. + +```bash +# Run `flutter build ios` on all packages that are different between current +# branch and the specified commit hash. +melos exec --diff= -- flutter build ios + +# Run `flutter build ios` on all packages that are different between remote +# `main` branch and HEAD. +melos exec --diff=origin/main...HEAD -- flutter build ios + +# Find all packages that are different between remote `main` branch and HEAD, +# add their dependents to the list, filter out packages without `test` +# directory and run `flutter test` on them. +melos exec --diff=origin/main...HEAD --include-dependents --reversed-filter-group-priority --dir-exists="test" -- flutter test +``` + +### --include-dependencies Takes the filtered list of packages, and expands them to include those packages' transitive dependencies (ignoring filters). @@ -112,7 +131,7 @@ transitive dependencies (ignoring filters). melos list --scope=some_package --include-dependencies ``` -## --include-dependents +### --include-dependents Takes the filtered list of packages, and expands them to include those packages' transitive dependents (ignoring filters). diff --git a/packages/melos/lib/src/command_runner/base.dart b/packages/melos/lib/src/command_runner/base.dart index 9f604d84..179e2c77 100644 --- a/packages/melos/lib/src/command_runner/base.dart +++ b/packages/melos/lib/src/command_runner/base.dart @@ -42,7 +42,8 @@ abstract class MelosCommand extends Command { argParser.addFlag( filterOptionPrivate, help: 'Whether to include or exclude packages with `publish_to: "none"`. ' - 'By default, the filter has no effect.', + 'By default, the filter has no effect.' + ' ${FilterGroup.belongsToFirstPriorityGroupFilterDescription}', defaultsTo: null, ); @@ -51,7 +52,8 @@ abstract class MelosCommand extends Command { defaultsTo: null, help: 'Filter packages where the current local package version exists on ' 'pub.dev. Or "-no-published" to filter packages that have not had ' - 'their current version published yet.', + 'their current version published yet.' + ' ${FilterGroup.belongsToFirstPriorityGroupFilterDescription}', ); argParser.addFlag( @@ -60,7 +62,8 @@ abstract class MelosCommand extends Command { help: 'Filter packages where the current local version uses a "nullsafety" ' 'prerelease preid. Or "-no-nullsafety" to filter packages where ' - 'their current version does not have a "nullsafety" preid.', + 'their current version does not have a "nullsafety" preid.' + ' ${FilterGroup.belongsToFirstPriorityGroupFilterDescription}', ); argParser.addFlag( @@ -68,44 +71,39 @@ abstract class MelosCommand extends Command { defaultsTo: null, help: 'Filter packages where the package depends on the Flutter SDK. Or ' '"-no-flutter" to filter packages that do not depend on the Flutter ' - 'SDK.', + 'SDK.' + ' ${FilterGroup.belongsToFirstPriorityGroupFilterDescription}', ); argParser.addMultiOption( filterOptionScope, valueHelp: 'glob', help: 'Include only packages with names matching the given glob. This ' - 'option can be repeated.', + 'option can be repeated.' + ' ${FilterGroup.belongsToFirstPriorityGroupFilterDescription}', ); argParser.addMultiOption( filterOptionIgnore, valueHelp: 'glob', help: 'Exclude packages with names matching the given glob. This option ' - 'can be repeated.', - ); - - argParser.addOption( - filterOptionDiff, - valueHelp: 'ref', - help: 'Filter packages based on whether there were changes between a ' - 'commit and the current HEAD or within a range of commits. A range ' - 'of commits can be specified using the git short hand syntax ' - '`..` and `...`', + 'can be repeated.' + ' ${FilterGroup.belongsToFirstPriorityGroupFilterDescription}', ); argParser.addMultiOption( filterOptionDirExists, valueHelp: 'dirRelativeToPackageRoot', help: 'Include only packages where a specific directory exists inside ' - 'the package.', + 'the package.' + ' ${FilterGroup.belongsToFirstPriorityGroupFilterDescription}', ); argParser.addMultiOption( filterOptionFileExists, valueHelp: 'fileRelativeToPackageRoot', - help: - 'Include only packages where a specific file exists in the package.', + help: 'Include only packages where a specific file exists in the package.' + ' ${FilterGroup.belongsToFirstPriorityGroupFilterDescription}', ); argParser.addMultiOption( @@ -122,20 +120,37 @@ abstract class MelosCommand extends Command { 'This option can be repeated.', ); + argParser.addOption( + filterOptionDiff, + valueHelp: 'ref', + help: 'Filter packages based on whether there were changes between a ' + 'commit and the current HEAD or within a range of commits. A range ' + 'of commits can be specified using the git short hand syntax ' + '`..` and `...`.' + ' ${FilterGroup.belongsToSecondPriorityGroupFilterDescription}', + ); + argParser.addFlag( filterOptionIncludeDependents, negatable: false, help: 'Include all transitive dependents for each package that matches ' - 'the other filters. The included packages skip --ignore and ' - '--diff checks.', + 'the other filters.' + ' ${FilterGroup.belongsToSecondPriorityGroupFilterDescription}', ); argParser.addFlag( filterOptionIncludeDependencies, negatable: false, help: 'Include all transitive dependencies for each package that ' - 'matches the other filters. The included packages skip --ignore ' - 'and --diff checks.', + 'matches the other filters.' + ' ${FilterGroup.belongsToSecondPriorityGroupFilterDescription}', + ); + + argParser.addFlag( + filterOptionReversedFilterGroupPriority, + negatable: false, + help: 'Filters are divided into several groups based on their priority ' + 'of execution. This flag reverses the priority of these groups.', ); } @@ -172,6 +187,8 @@ abstract class MelosCommand extends Command { noDependsOn: argResults![filterOptionNoDependsOn] as List? ?? [], includeDependents: argResults![filterOptionIncludeDependents] as bool, includeDependencies: argResults![filterOptionIncludeDependencies] as bool, + reversedFilterGroupPriority: + argResults![filterOptionReversedFilterGroupPriority] as bool, ); } } diff --git a/packages/melos/lib/src/common/utils.dart b/packages/melos/lib/src/common/utils.dart index bcc8a4ac..95b2b0b8 100644 --- a/packages/melos/lib/src/common/utils.dart +++ b/packages/melos/lib/src/common/utils.dart @@ -39,6 +39,8 @@ const filterOptionDependsOn = 'depends-on'; const filterOptionNoDependsOn = 'no-depends-on'; const filterOptionIncludeDependents = 'include-dependents'; const filterOptionIncludeDependencies = 'include-dependencies'; +const filterOptionReversedFilterGroupPriority = + 'reversed-filter-group-priority'; const publishOptionDryRun = 'dry-run'; const publishOptionNoDryRun = 'no-dry-run'; diff --git a/packages/melos/lib/src/package.dart b/packages/melos/lib/src/package.dart index 2e0fc4de..4aa9c802 100644 --- a/packages/melos/lib/src/package.dart +++ b/packages/melos/lib/src/package.dart @@ -112,6 +112,7 @@ class PackageFilters { bool? flutter, this.includeDependencies = false, this.includeDependents = false, + this.reversedFilterGroupPriority = false, }) : dependsOn = [ ...dependsOn, // ignore: use_if_null_to_convert_nulls_to_bools @@ -230,6 +231,13 @@ class PackageFilters { path: path, ); + final reversedFilterGroupPriority = assertKeyIsA( + key: filterOptionFlutter.camelCased, + map: yaml, + path: path, + ) ?? + false; + Glob createPackageGlob(String pattern) => createGlob(pattern, currentDirectoryPath: workspacePath); @@ -247,6 +255,7 @@ class PackageFilters { published: published, nullSafe: nullSafe, flutter: flutter, + reversedFilterGroupPriority: reversedFilterGroupPriority, ); } @@ -265,6 +274,7 @@ class PackageFilters { required this.nullSafe, required this.includeDependencies, required this.includeDependents, + required this.reversedFilterGroupPriority, }); /// Patterns for filtering packages by name. @@ -311,6 +321,11 @@ class PackageFilters { /// This supersede other filters. final bool includeDependencies; + /// All the filters are divided into several groups, they applied sequentially + /// based on the priority of the group. This flag allows to reverse the order + /// of the groups, so that the filters from the last group are applied first. + final bool reversedFilterGroupPriority; + Map toJson() { return { if (scope.isNotEmpty) @@ -329,6 +344,8 @@ class PackageFilters { if (nullSafe != null) filterOptionNullsafety.camelCased: nullSafe, if (includeDependents) filterOptionIncludeDependents.camelCased: true, if (includeDependencies) filterOptionIncludeDependencies.camelCased: true, + if (reversedFilterGroupPriority) + filterOptionReversedFilterGroupPriority.camelCased: true, }; } @@ -346,6 +363,7 @@ class PackageFilters { diff: diff, includeDependencies: includeDependencies, includeDependents: includeDependents, + reversedFilterGroupPriority: reversedFilterGroupPriority, ); } @@ -363,6 +381,7 @@ class PackageFilters { diff: diff, includeDependencies: includeDependencies, includeDependents: includeDependents, + reversedFilterGroupPriority: reversedFilterGroupPriority, ); } @@ -379,6 +398,7 @@ class PackageFilters { String? diff, bool? includeDependencies, bool? includeDependents, + bool? reversedFilterGroupPriority, }) { return PackageFilters._( dependsOn: dependsOn ?? this.dependsOn, @@ -394,6 +414,8 @@ class PackageFilters { diff: diff ?? this.diff, includeDependencies: includeDependencies ?? this.includeDependencies, includeDependents: includeDependents ?? this.includeDependents, + reversedFilterGroupPriority: + reversedFilterGroupPriority ?? this.reversedFilterGroupPriority, ); } @@ -412,7 +434,8 @@ class PackageFilters { const DeepCollectionEquality().equals(other.fileExists, fileExists) && const DeepCollectionEquality().equals(other.dependsOn, dependsOn) && const DeepCollectionEquality().equals(other.noDependsOn, noDependsOn) && - other.diff == diff; + other.diff == diff && + other.reversedFilterGroupPriority == reversedFilterGroupPriority; @override int get hashCode => @@ -428,7 +451,8 @@ class PackageFilters { const DeepCollectionEquality().hash(fileExists) ^ const DeepCollectionEquality().hash(dependsOn) ^ const DeepCollectionEquality().hash(noDependsOn) ^ - diff.hashCode; + diff.hashCode ^ + reversedFilterGroupPriority.hashCode; @override String toString() { @@ -446,6 +470,7 @@ PackageFilters( dependsOn: $dependsOn, noDependsOn: $noDependsOn, diff: $diff, + reversedFilterGroupPriority: $reversedFilterGroupPriority, )'''; } } @@ -596,30 +621,54 @@ The packages that caused the problem are: Future applyFilters(PackageFilters? filters) async { if (filters == null) return this; - var packageList = await values - .applyIgnore(filters.ignore) - .applyDirExists(filters.dirExists) - .applyFileExists(filters.fileExists) - .filterPrivatePackages(include: filters.includePrivatePackages) - .applyScope(filters.scope) - .applyDependsOn(filters.dependsOn) - .applyNoDependsOn(filters.noDependsOn) - .filterNullSafe(nullSafe: filters.nullSafe) - .filterPublishedPackages(published: filters.published); - - final diff = filters.diff; - if (diff != null) { - packageList = await packageList.applyDiff(diff, _logger); - } + final filterGroups = [ + FilterGroup.firstPriority( + filters: [ + (packages) => Future.value(packages.applyIgnore(filters.ignore)), + (packages) => + Future.value(packages.applyDirExists(filters.dirExists)), + (packages) => + Future.value(packages.applyFileExists(filters.fileExists)), + (packages) => Future.value( + packages.filterPrivatePackages( + include: filters.includePrivatePackages, + ), + ), + (packages) => Future.value(packages.applyScope(filters.scope)), + (packages) => + Future.value(packages.applyDependsOn(filters.dependsOn)), + (packages) => + Future.value(packages.applyNoDependsOn(filters.noDependsOn)), + (packages) => + Future.value(packages.filterNullSafe(nullSafe: filters.nullSafe)), + (packages) => + packages.filterPublishedPackages(published: filters.published), + ], + ), + FilterGroup.secondPriority( + filters: [ + (packages) => packages.applyDiff(filters.diff, _logger), + (packages) => Future.value( + packages.applyIncludeDependentsOrDependencies( + includeDependents: filters.includeDependents, + includeDependencies: filters.includeDependencies, + ), + ), + ], + ), + ]; - packageList = packageList.applyIncludeDependentsOrDependencies( - includeDependents: filters.includeDependents, - includeDependencies: filters.includeDependencies, - ); + var filteredPackages = values; + + for (final filterGroup in (filters.reversedFilterGroupPriority + ? filterGroups.reversed + : filterGroups)) { + filteredPackages = await filterGroup.apply(filteredPackages); + } return PackageMap( { - for (final package in packageList) package.name: package, + for (final package in filteredPackages) package.name: package, }, _logger, ); @@ -1134,3 +1183,47 @@ class Plugin { Map? get platforms => _plugin['platforms'] as Map?; } + +enum FilterPriority { + first, + second, +} + +typedef FilterFunction = Future> Function( + Iterable packages, +); + +class FilterGroup { + const FilterGroup.firstPriority({ + required List filters, + }) : this._(priority: FilterPriority.first, filters: filters); + + const FilterGroup.secondPriority({ + required List filters, + }) : this._(priority: FilterPriority.second, filters: filters); + + const FilterGroup._({ + required this.priority, + required this.filters, + }); + + static const String belongsToFirstPriorityGroupFilterDescription = + 'The filter belongs to 1st priority group.'; + + static const String belongsToSecondPriorityGroupFilterDescription = + 'The filter belongs to 2nd priority group.'; + + final FilterPriority priority; + final List filters; + + /// Sequentially applies all [filters] in this group to the given [packages]. + Future> apply(Iterable packages) async { + var filtered = packages; + + for (final filter in filters) { + filtered = await filter(filtered); + } + + return filtered; + } +} diff --git a/packages/melos/test/package_test.dart b/packages/melos/test/package_test.dart index ae8b6fd6..3b2ca94e 100644 --- a/packages/melos/test/package_test.dart +++ b/packages/melos/test/package_test.dart @@ -257,6 +257,7 @@ void main() { expect(filters.nullSafe, null); expect(filters.published, null); expect(filters.diff, null); + expect(filters.reversedFilterGroupPriority, false); }); group('copyWithWithDiff', () { @@ -281,6 +282,7 @@ void main() { nullSafe: true, published: true, diff: '123', + reversedFilterGroupPriority: true, ); final copy = filters.copyWithDiff('456'); @@ -297,9 +299,93 @@ void main() { expect(copy.noDependsOn, filters.noDependsOn); expect(copy.nullSafe, filters.nullSafe); expect(copy.published, filters.published); + expect( + copy.reversedFilterGroupPriority, + filters.reversedFilterGroupPriority, + ); }); }); }); + + group('PackageMap', () { + test('applyFilters works with original filter group priority', () async { + final workspaceBuilder = VirtualWorkspaceBuilder('name: test'); + workspaceBuilder.addPackage(''' + name: a + '''); + workspaceBuilder.addPackage(''' + name: b + dependencies: + a: any + '''); + workspaceBuilder.addPackage(''' + name: c + dependencies: + b: any + '''); + workspaceBuilder.addPackage(''' + name: d + '''); + final workspace = workspaceBuilder.build(); + + final packageMap = workspace.allPackages; + + final filters = PackageFilters( + dependsOn: const ['a'], + includeDependents: true, + ); + + final filteredPackages = await packageMap.applyFilters(filters); + + expect(filteredPackages.length, 2); + expect( + filteredPackages.values, + [ + isA().having((p) => p.name, 'name', 'b'), + isA().having((p) => p.name, 'name', 'c'), + ], + ); + }); + + test('applyFilters works with reversed filter group priority', () async { + final workspaceBuilder = VirtualWorkspaceBuilder('name: test'); + workspaceBuilder.addPackage(''' + name: a + '''); + workspaceBuilder.addPackage(''' + name: b + dependencies: + a: any + '''); + workspaceBuilder.addPackage(''' + name: c + dependencies: + b: any + '''); + workspaceBuilder.addPackage(''' + name: d + '''); + final workspace = workspaceBuilder.build(); + + final packageMap = workspace.allPackages; + + final filters = PackageFilters( + dependsOn: const ['a'], + includeDependents: true, + reversedFilterGroupPriority: true, + ); + + final filteredPackages = await packageMap.applyFilters(filters); + + expect(filteredPackages.length, 1); + expect( + filteredPackages.values, + [ + isA().having((p) => p.name, 'name', 'b'), + ], + ); + }); + }); } void testDependencyVersionReplaceRegex(String version) {