Merge pull request #179 from buildkite-plugins/toote_min_tests

Min tests flag

README.md

@@ -14,7 +14,7 @@ steps:
   - wait: ~
     continue_on_failure: true
   - plugins:
-      - junit-annotate#v2.1.0:
+      - junit-annotate#v2.2.0:
           artifacts: tmp/junit-*.xml
 ```
 
@@ -30,37 +30,42 @@ Example: `tmp/junit-*.xml`
 
 Forces the creation of the annotation even when no failures or errors are found
 
+### `context` (optional)
+
+Default: `junit`
+
+The buildkite annotation context to use. Useful to differentiate multiple runs of this plugin in a single pipeline.
+
 ### `job-uuid-file-pattern` (optional)
+
 Default: `-(.*).xml`
 
-The regular expression (with capture group) that matches the job UUID in the junit file names. This is used to create the job links in the annotation.
+The regular expression (with capture group) that matches the job UUID in the junit file names. This is used to create the job links in the annotation. 
 
 To use this, configure your test reporter to embed the `$BUILDKITE_JOB_ID` environment variable into your junit file names. For example `"junit-buildkite-job-$BUILDKITE_JOB_ID.xml"`.
 
 ### `failure-format` (optional)
-Default: `classname`
 
 This setting controls the format of your failed test in the main annotation summary.
 
 There are two options for this:
-* `classname`
+* `classname` (the default)
   * displays: `MyClass::UnderTest text of the failed expectation in path.to.my_class.under_test`
 * `file`
   * displays: `MyClass::UnderTest text of the failed expectation in path/to/my_class/under_test.file_ext`
 
 ### `fail-build-on-error` (optional)  
+
 Default: `false`
 
-If this setting is true and any errors are found in the JUnit XML files during
-parsing, the annotation step will exit with a non-zero value, which should cause 
-the build to fail.
+If this setting is true and any errors are found in the JUnit XML files during parsing, the annotation step will exit with a non-zero value, which should cause the build to fail.
 
-### `context` (optional)
-Default: `junit`
+### `min-tests` (optional, integer)
 
-The buildkite annotation context to use. Useful to differentiate multiple runs of this plugin in a single pipeline.
+Minimum amount of run tests that need to be analyzed or a failure will be reported. It is useful to ensure that tests are actually run and report files to analyze do contain information.
 
 ### `report-slowest` (optional)
+
 Default: `0`
 
 Include the specified number of slowest tests in the annotation. The annotation will always be shown.
@@ -87,7 +92,7 @@ To test your plugin in your builds prior to opening a pull request, you can refe
 steps:
   - label: Annotate
     plugins:
-      - YourGithubHandle/junit-annotate#v2.1.0:
+      - YourGithubHandle/junit-annotate#v2.2.0:
           ...
 ```
 

hooks/command

@@ -76,6 +76,18 @@ if [ $has_errors -eq 0 ]; then
     echo "Will create annotation anyways"
     create_annotation=1
   fi
+
+  if [[ -e "${annotation_path}" ]]; then
+    TOTAL_TESTS=$(head -4 "${annotation_path}" | grep 'Total tests' | cut -d\  -f3)
+  else
+    TOTAL_TESTS=0
+  fi
+
+  if [[ "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_MIN_TESTS:-0}" -gt "${TOTAL_TESTS}" ]]; then
+    create_annotation=1
+    fail_build=1
+    echo ":warning: Less than ${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_MIN_TESTS} tests analyzed"
+  fi
 elif ! check_size; then
   echo "--- :warning: Failures too large to annotate"
 

plugin.yml

@@ -20,6 +20,8 @@ configuration:
       type: boolean
     job-uuid-file-pattern:
       type: string
+    min-tests:
+      type: integer
     report-slowest:
       type: integer
   required:

tests/command.bats

@@ -53,7 +53,7 @@ export annotation_input="tests/tmp/annotation.input"
     "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved"
 
   stub docker \
-    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '<details>Failure</details>' && exit 64"
+    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64"
 
   run "$PWD/hooks/command"
 
@@ -80,7 +80,7 @@ export annotation_input="tests/tmp/annotation.input"
     "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved"
 
   stub docker \
-    "--log-level error run --rm --volume \* --volume \* --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN='custom_(*)_pattern.xml' --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '<details>Failure</details>' && exit 64"
+    "--log-level error run --rm --volume \* --volume \* --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN='custom_(*)_pattern.xml' --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64"
 
   run "$PWD/hooks/command"
 
@@ -107,7 +107,7 @@ export annotation_input="tests/tmp/annotation.input"
     "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved"
 
   stub docker \
-    "--log-level error run --rm --volume \* --volume \* --env \* --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT='file' --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '<details>Failure</details>' && exit 64"
+    "--log-level error run --rm --volume \* --volume \* --env \* --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT='file' --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64"
 
   run "$PWD/hooks/command"
 
@@ -132,7 +132,7 @@ export annotation_input="tests/tmp/annotation.input"
     "artifact download \* \* : echo Downloaded artifact \$3 to \$4"
 
   stub docker \
-    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo No test errors"
+    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo 'Total tests: 0'"
 
   run "$PWD/hooks/command"
 
@@ -156,13 +156,14 @@ export annotation_input="tests/tmp/annotation.input"
     "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved"
 
   stub docker \
-    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo No test errors"
+    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo 'Total tests: 0'"
 
   run "$PWD/hooks/command"
 
   assert_success
-  assert_output --partial "No test errors"
+  assert_output --partial "Total tests: 0"
   assert_output --partial "Will create annotation anyways"
+  assert_equal "$(cat "${annotation_input}")" 'Total tests: 0'
 
   unstub mktemp
   unstub buildkite-agent
@@ -194,7 +195,7 @@ export annotation_input="tests/tmp/annotation.input"
     "artifact download \* \* : echo Downloaded artifact \$3 to \$4"
 
   stub docker \
-    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '<details>Failure</details>' && exit 64"
+    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64"
 
   run "$PWD/hooks/command"
 
@@ -256,7 +257,7 @@ export annotation_input="tests/tmp/annotation.input"
     "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved"
 
   stub docker \
-    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '<details>Failure</details>' && exit 64"
+    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64"
 
   run "$PWD/hooks/command"
 
@@ -285,7 +286,7 @@ export annotation_input="tests/tmp/annotation.input"
     "artifact download \* \* : echo Downloaded artifact \$3 to \$4"
 
   stub docker \
-    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '<details>Failure</details>' && exit 64"
+    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64"
 
   run "$PWD/hooks/command"
 
@@ -311,7 +312,7 @@ export annotation_input="tests/tmp/annotation.input"
     "artifact download \* \* : echo Downloaded artifact \$3 to \$4"
 
   stub docker \
-    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '<details>Failure</details>' && exit 147"
+    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 147"
 
   run "$PWD/hooks/command"
 
@@ -343,4 +344,81 @@ export annotation_input="tests/tmp/annotation.input"
 
   unstub mktemp
   unstub buildkite-agent
+}
+
+@test "creates annotation with no failures but min tests triggers" {
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml"
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_MIN_TESTS=1
+
+  stub mktemp \
+    "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \
+    "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'"
+
+  stub buildkite-agent \
+    "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \
+    "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved"
+
+  stub docker \
+    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo 'Total tests: 0'"
+
+  run "$PWD/hooks/command"
+
+  assert_failure
+  assert_output --partial "Total tests: 0"
+  assert_output --partial "Less than 1 tests analyzed"
+  assert_equal "$(cat "${annotation_input}")" 'Total tests: 0'
+
+  unstub mktemp
+  unstub buildkite-agent
+  unstub docker
+}
+
+@test "no failures and min-tests ok does not create annotation" {
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml"
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_MIN_TESTS=12
+
+  stub mktemp \
+    "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \
+    "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'"
+
+  stub buildkite-agent \
+    "artifact download \* \* : echo Downloaded artifact \$3 to \$4"
+
+  stub docker \
+    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo 'Total tests: 100'"
+
+  run "$PWD/hooks/command"
+
+  assert_success
+  assert_output --partial "Total tests: 100"
+  refute_output --partial "Less than 12 tests analyzed"
+
+  unstub mktemp
+  unstub buildkite-agent
+  unstub docker
+}
+
+@test "min-tests doesn't interfere with actual failures" {
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml"
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_MIN_TESTS=10000
+
+  stub mktemp \
+    "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \
+    "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'"
+
+  stub buildkite-agent \
+    "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \
+    "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved"
+
+  stub docker \
+    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64"
+
+  run "$PWD/hooks/command"
+
+  assert_success
+  assert_output --partial "Total tests: 2"
+
+  unstub mktemp
+  unstub buildkite-agent
+  unstub docker
 }