diff --git a/README.md b/README.md
index 55eb5f7..3dafbd5 100644
--- a/README.md
+++ b/README.md
@@ -23,6 +23,10 @@ The local directory holding the JUnit XML files.
 
 Example: `tmp/junit-*.xml`
 
+### `always-annotate` (optional, boolean)
+
+Forces the creation of the annotation even when no failures or errors are found
+
 ### `job-uuid-file-pattern` (optional)
 Default: `-(.*).xml`
 
diff --git a/docker-compose.yml b/docker-compose.yml
index 15eb297..14f7e86 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -6,13 +6,13 @@ services:
     volumes:
       - ".:/plugin"
   plugin:
-    image: buildkite/plugin-tester:latest@sha256:476a1024936901889147f53d2a3d8e71e99d76404972d583825514f5608083dc
+    image: buildkite/plugin-tester:v3.0.1
     volumes:
       - ".:/plugin"
     depends_on:
       - ruby
   ruby:
-    image: ruby:3.1-alpine@sha256:0b91e98c4613c2287bb7274a1584ea141b68960fb6b0edd7fe32127dc888d1a0
+    image: ruby:3.1-alpine@sha256:c5acbb8bcc57cc3cb8da7f28077ec23c9c05217f26bd4e156d7b87df6dcf0c00
     command: rake
     working_dir: /src
     volumes:
diff --git a/hooks/post-command b/hooks/post-command
index 65877c5..81e60dc 100755
--- a/hooks/post-command
+++ b/hooks/post-command
@@ -15,6 +15,8 @@ annotation_dir="$(pwd)/$(mktemp -d "junit-annotate-plugin-annotation-tmp.XXXXXXX
 annotation_path="${annotation_dir}/annotation.md"
 annotation_style="info"
 fail_build=0
+has_errors=0
+create_annotation=0
 
 function cleanup {
   rm -rf "${annotation_dir}"
@@ -45,39 +47,58 @@ docker \
     ruby:2.7-alpine ruby /src/bin/annotate /junits \
       > "$annotation_path"
 
-if [[ $? -eq 64 ]]; then # special exit code to signal test failures
+exit_code=$?
+set -e
+
+if [[ $exit_code -eq 64 ]]; then # special exit code to signal test failures
+  has_errors=1
+  create_annotation=1
   annotation_style="error"
   if [[ "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_FAIL_BUILD_ON_ERROR:-false}" =~ (true|on|1) ]]; then
+    echo "--- :boom: Build will fail due to errors being found"
     fail_build=1
   fi
+elif [[ $exit_code -ne 0 ]]; then
+  echo "--- :boom: Error when processing JUnit tests"
+  exit $exit_code
 fi
 
-set -e
-
 cat "$annotation_path"
 
-if grep -q "<details>" "$annotation_path"; then
+if [ $has_errors -eq 0 ]; then
+  # done in nested if to simplify outer conditions
+  if [[ "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_ALWAYS_ANNOTATE:-false}" =~ (true|on|1) ]]; then
+    echo "Will create annotation anyways"
+    create_annotation=1
+  fi
+elif ! check_size; then
+  echo "--- :warning: Failures too large to annotate"
+
+  # creating a simplified version of the annotation
+  mv "${annotation_path}" "${annotation_path}2"
+  head -4 "${annotation_path}2" >"${annotation_path}"
+  # || true is to avoid issues if no summary is found
+  grep '<summary>' "${annotation_path}2" >>"${annotation_path}" || true
+
   if ! check_size; then
-    echo "--- :warning: Failures too large to annotate"
-    msg="The failures are too large to create a build annotation. Please inspect the failed JUnit artifacts manually."
-    echo "$msg"
+    echo "The failures are too large to create a build annotation. Please inspect the failed JUnit artifacts manually."
+    create_annotation=0
   else
-    echo "--- :buildkite: Creating annotation"
-    args=(
-      --context "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_CONTEXT:-junit}"
-      --style "$annotation_style"
-    )
-    if [[ "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_APPEND:-false}" =~ (true|on|1) ]]; then
-      args+=( --append )
-    fi
-    # shellcheck disable=SC2002
-    cat "$annotation_path" | buildkite-agent annotate "${args[@]}"
+    echo "The failures are too large to create complete annotation, using a simplified annotation"
   fi
 fi
 
-if ((fail_build)); then
-  echo "--- :boom: Failing build due to error"
-  exit 1
-else
-  exit 0
+if [ $create_annotation -ne 0 ]; then
+  echo "--- :buildkite: Creating annotation"
+  args=(
+    --context "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_CONTEXT:-junit}"
+    --style "$annotation_style"
+  )
+  if [[ "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_APPEND:-false}" =~ (true|on|1) ]]; then
+    args+=( --append )
+  fi
+  # shellcheck disable=SC2002
+  cat "$annotation_path" | buildkite-agent annotate "${args[@]}"
 fi
+
+exit $fail_build
diff --git a/plugin.yml b/plugin.yml
index 46b1f4c..eecd809 100644
--- a/plugin.yml
+++ b/plugin.yml
@@ -7,7 +7,9 @@ configuration:
   properties:
     directory:
       type: string
-    job-uuid-file-pattern:
+    always-annotate:
+      type: boolean
+    context:
       type: string
     failure-format:
       type: string
@@ -16,7 +18,7 @@ configuration:
         - file
     fail-build-on-error:
       type: boolean
-    context:
+    job-uuid-file-pattern:
       type: string
     report-slowest:
       type: integer
diff --git a/renovate.json b/renovate.json
index 1c4999c..69e6dc4 100644
--- a/renovate.json
+++ b/renovate.json
@@ -1,6 +1,7 @@
 {
   "extends": [
-    "config:base"
+    "config:base",
+    ":disableDependencyDashboard"
   ],
   "bundler": {
     "enabled": true
diff --git a/ruby/bin/annotate b/ruby/bin/annotate
index f216247..62d063c 100755
--- a/ruby/bin/annotate
+++ b/ruby/bin/annotate
@@ -11,13 +11,13 @@ junits_dir = ARGV[0]
 abort("Usage: annotate <junits-dir>") unless junits_dir
 abort("#{junits_dir} does not exist") unless Dir.exist?(junits_dir)
 
-job_pattern = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN']
+job_pattern = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_JOB_UUID_FILE_PATTERN']
 job_pattern = '-(.*).xml' if !job_pattern || job_pattern.empty?
 
-failure_format = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT']
+failure_format = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_FAILURE_FORMAT']
 failure_format = 'classname' if !failure_format || failure_format.empty?
 
-report_slowest = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST'].to_i
+report_slowest = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_REPORT_SLOWEST'].to_i
 
 class Failure < Struct.new(:name, :unit_name, :body, :job, :type, :message)
 end
@@ -74,41 +74,33 @@ junit_report_files.sort.each do |file|
 end
 
 STDERR.puts "--- ✍️ Preparing annotation"
-STDERR.puts "#{testcases} testcases found"
-
-if failures.any?
-  STDERR.puts "There #{failures.length == 1 ? "is 1 failure/error" : "are #{failures.length} failures/errors" } 😭"
-
-  failures_count = failures.select {|f| f.type == :failure }.length
-  errors_count = failures.select {|f| f.type == :error }.length
-  puts [
-    failures_count == 0 ? nil : (failures_count == 1 ? "1 failure" : "#{failures_count} failures"),
-    errors_count === 0 ? nil : (errors_count == 1 ? "1 error" : "#{errors_count} errors"),
-  ].compact.join(" and ") + ":\n\n"
-
-  failures.each do |failure|
-    puts "<details>"
-    puts "<summary><code>#{failure.name} in #{failure.unit_name}</code></summary>\n\n"
-    if failure.message
-      puts "<p>#{failure.message.chomp.strip}</p>\n\n"
-    end
-    if failure.body
-      puts "<pre><code>#{CGI.escapeHTML(failure.body.chomp.strip)}</code></pre>\n\n"
-    end
-    if failure.job
-      puts "in <a href=\"##{failure.job}\">Job ##{failure.job}</a>"
-    end
-    puts "</details>"
-    puts "" unless failure == failures.last
-  end
 
-else
-  STDERR.puts "There were no failures/errors 🙌"
+failures_count = failures.select {|f| f.type == :failure }.length
+errors_count = failures.select {|f| f.type == :error }.length
+
+puts "Failures: #{failures_count}"
+puts "Errors: #{errors_count}"
+puts "Total tests: #{testcases}"
+
+failures.each do |failure|
+  puts ""
+  puts "<details>"
+  puts "<summary><code>#{failure.name} in #{failure.unit_name}</code></summary>\n\n"
+  if failure.message
+    puts "<p>#{failure.message.chomp.strip}</p>\n\n"
+  end
+  if failure.body
+    puts "<pre><code>#{CGI.escapeHTML(failure.body.chomp.strip)}</code></pre>\n\n"
+  end
+  if failure.job
+    puts "in <a href=\"##{failure.job}\">Job ##{failure.job}</a>"
+  end
+  puts "</details>"
 end
 
 if report_slowest > 0
   STDERR.puts "Reporting slowest tests ⏱"
-
+  puts ""
   puts "<details>"
   puts "<summary>#{report_slowest} slowest tests</summary>\n\n"
   puts "<table>"
diff --git a/ruby/tests/annotate_test.rb b/ruby/tests/annotate_test.rb
index d42ac65..0a93091 100644
--- a/ruby/tests/annotate_test.rb
+++ b/ruby/tests/annotate_test.rb
@@ -3,31 +3,38 @@
 
 describe "Junit annotate plugin parser" do
   it "handles no failures" do
-    output, status = Open3.capture2e("#{__dir__}/../bin/annotate", "#{__dir__}/no-test-failures/")
+    stdout, stderr, status = Open3.capture3("#{__dir__}/../bin/annotate", "#{__dir__}/no-test-failures/")
 
-    assert_equal <<~OUTPUT, output
+    assert_equal stderr, <<~OUTPUT
       Parsing junit-1.xml
       Parsing junit-2.xml
       Parsing junit-3.xml
       --- ✍️ Preparing annotation
-      8 testcases found
-      There were no failures/errors 🙌
+    OUTPUT
+
+    assert_equal stdout, <<~OUTPUT
+      Failures: 0
+      Errors: 0
+      Total tests: 8
     OUTPUT
 
     assert_equal 0, status.exitstatus
   end
 
   it "handles failures across multiple files" do
-    output, status = Open3.capture2e("#{__dir__}/../bin/annotate", "#{__dir__}/two-test-failures/")
+    stdout, stderr, status = Open3.capture3("#{__dir__}/../bin/annotate", "#{__dir__}/two-test-failures/")
 
-    assert_equal <<~OUTPUT, output
+    assert_equal stderr, <<~OUTPUT
       Parsing junit-1.xml
       Parsing junit-2.xml
       Parsing junit-3.xml
       --- ✍️ Preparing annotation
-      6 testcases found
-      There are 4 failures/errors 😭
-      4 failures:
+    OUTPUT
+
+    assert_equal stdout, <<~OUTPUT
+      Failures: 4
+      Errors: 0
+      Total tests: 6
       
       <details>
       <summary><code>Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec</code></summary>
@@ -110,17 +117,20 @@
   end
 
   it "handles failures and errors across multiple files" do
-    output, status = Open3.capture2e("#{__dir__}/../bin/annotate", "#{__dir__}/test-failure-and-error/")
+    stdout, stderr, status = Open3.capture3("#{__dir__}/../bin/annotate", "#{__dir__}/test-failure-and-error/")
 
-    assert_equal <<~OUTPUT, output
+    assert_equal stderr, <<~OUTPUT
       Parsing junit-1.xml
       Parsing junit-2.xml
       Parsing junit-3.xml
       --- ✍️ Preparing annotation
-      6 testcases found
-      There are 4 failures/errors 😭
-      2 failures and 2 errors:
-      
+    OUTPUT
+
+    assert_equal stdout, <<~OUTPUT
+      Failures: 2
+      Errors: 2
+      Total tests: 6
+
       <details>
       <summary><code>Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec</code></summary>
       
@@ -202,14 +212,17 @@
   end
 
   it "accepts custom regex filename patterns for job id" do
-    output, status = Open3.capture2e("env", "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_JOB_UUID_FILE_PATTERN=junit-(.*)-custom-pattern.xml", "#{__dir__}/../bin/annotate", "#{__dir__}/custom-job-uuid-pattern/")
+    stdout, stderr, status = Open3.capture3("env", "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_JOB_UUID_FILE_PATTERN=junit-(.*)-custom-pattern.xml", "#{__dir__}/../bin/annotate", "#{__dir__}/custom-job-uuid-pattern/")
 
-    assert_equal <<~OUTPUT, output
+    assert_equal stderr, <<~OUTPUT
       Parsing junit-123-456-custom-pattern.xml
       --- ✍️ Preparing annotation
-      2 testcases found
-      There is 1 failure/error 😭
-      1 failure:
+    OUTPUT
+
+    assert_equal stdout, <<~OUTPUT
+      Failures: 1
+      Errors: 0
+      Total tests: 2
       
       <details>
       <summary><code>Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec</code></summary>
@@ -235,16 +248,20 @@
   end
 
   it "uses the file path instead of classname for annotation content when specified" do
-    output, status = Open3.capture2e("env", "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_FAILURE_FORMAT=file", "#{__dir__}/../bin/annotate", "#{__dir__}/test-failure-and-error/")
 
-    assert_equal <<~OUTPUT, output
+    stdout, stderr, status = Open3.capture3("env", "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_FAILURE_FORMAT=file", "#{__dir__}/../bin/annotate", "#{__dir__}/test-failure-and-error/")
+
+    assert_equal stderr, <<~OUTPUT
       Parsing junit-1.xml
       Parsing junit-2.xml
       Parsing junit-3.xml
       --- ✍️ Preparing annotation
-      6 testcases found
-      There are 4 failures/errors 😭
-      2 failures and 2 errors:
+    OUTPUT
+
+    assert_equal stdout, <<~OUTPUT
+      Failures: 2
+      Errors: 2
+      Total tests: 6
 
       <details>
       <summary><code>Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in ./spec/models/account_spec.rb</code></summary>
@@ -327,16 +344,19 @@
   end
 
   it "handles failures across multiple files in sub dirs" do
-    output, status = Open3.capture2e("#{__dir__}/../bin/annotate", "#{__dir__}/tests-in-sub-dirs/")
+    stdout, stderr, status = Open3.capture3("#{__dir__}/../bin/annotate", "#{__dir__}/tests-in-sub-dirs/")
 
-    assert_equal <<~OUTPUT, output
+    assert_equal stderr, <<~OUTPUT
       Parsing sub-dir/junit-1.xml
       Parsing sub-dir/junit-2.xml
       Parsing sub-dir/junit-3.xml
       --- ✍️ Preparing annotation
-      6 testcases found
-      There are 4 failures/errors 😭
-      4 failures:
+    OUTPUT
+
+    assert_equal stdout, <<~OUTPUT
+      Failures: 4
+      Errors: 0
+      Total tests: 6
       
       <details>
       <summary><code>Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec</code></summary>
@@ -419,14 +439,17 @@
   end
 
   it "handles empty failure bodies" do
-    output, status = Open3.capture2e("#{__dir__}/../bin/annotate", "#{__dir__}/empty-failure-body/")
+    stdout, stderr, status = Open3.capture3("#{__dir__}/../bin/annotate", "#{__dir__}/empty-failure-body/")
 
-    assert_equal <<~OUTPUT, output
+    assert_equal stderr, <<~OUTPUT
       Parsing junit.xml
       --- ✍️ Preparing annotation
-      2 testcases found
-      There is 1 failure/error 😭
-      1 failure:
+    OUTPUT
+
+    assert_equal stdout, <<~OUTPUT
+      Failures: 1
+      Errors: 0
+      Total tests: 2
 
       <details>
       <summary><code>Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec</code></summary>
@@ -439,15 +462,18 @@
     assert_equal 64, status.exitstatus
   end
 
-  it "handles missing message attributes" do
-    output, status = Open3.capture2e("#{__dir__}/../bin/annotate", "#{__dir__}/missing-message-attribute/")
+  it "handles miss message attributes" do
+    stdout, stderr, status = Open3.capture3("#{__dir__}/../bin/annotate", "#{__dir__}/missing-message-attribute/")
 
-    assert_equal <<~OUTPUT, output
+    assert_equal stderr, <<~OUTPUT
       Parsing junit.xml
       --- ✍️ Preparing annotation
-      4 testcases found
-      There are 3 failures/errors 😭
-      1 failure and 2 errors:
+    OUTPUT
+
+    assert_equal stdout, <<~OUTPUT
+      Failures: 1
+      Errors: 2
+      Total tests: 4
 
       <details>
       <summary><code>Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec</code></summary>
@@ -469,14 +495,17 @@
   end
 
   it "handles cdata formatted XML files" do
-    output, status = Open3.capture2e("#{__dir__}/../bin/annotate", "#{__dir__}/failure-with-cdata/")
+    stdout, stderr, status = Open3.capture3("#{__dir__}/../bin/annotate", "#{__dir__}/failure-with-cdata/")
 
-    assert_equal <<~OUTPUT, output
+    assert_equal stderr, <<~OUTPUT
       Parsing junit.xml
       --- ✍️ Preparing annotation
-      2 testcases found
-      There is 1 failure/error 😭
-      1 error:
+    OUTPUT
+
+    assert_equal stdout, <<~OUTPUT
+      Failures: 0
+      Errors: 1
+      Total tests: 2
 
       <details>
       <summary><code>Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec</code></summary>
@@ -493,16 +522,21 @@
   end
 
   it "reports specified amount of slowest tests" do
-    output, status = Open3.capture2e("env", "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_REPORT_SLOWEST=5", "#{__dir__}/../bin/annotate", "#{__dir__}/no-test-failures/")
 
-    assert_equal <<~OUTPUT, output
+    stdout, stderr, status = Open3.capture3("env", "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_REPORT_SLOWEST=5", "#{__dir__}/../bin/annotate", "#{__dir__}/no-test-failures/")
+    assert_equal stderr, <<~OUTPUT
       Parsing junit-1.xml
       Parsing junit-2.xml
       Parsing junit-3.xml
       --- ✍️ Preparing annotation
-      8 testcases found
-      There were no failures/errors 🙌
       Reporting slowest tests ⏱
+    OUTPUT
+
+    assert_equal stdout, <<~OUTPUT
+      Failures: 0
+      Errors: 0
+      Total tests: 8
+      
       <details>
       <summary>5 slowest tests</summary>
 
@@ -523,13 +557,17 @@
   end
 
   it "handles junit dir paths with hidden directories" do
-    output, status = Open3.capture2e("#{__dir__}/../bin/annotate", "#{__dir__}/.tests-in-hidden-dir/")
+    stdout, stderr, status = Open3.capture3("#{__dir__}/../bin/annotate", "#{__dir__}/.tests-in-hidden-dir/")
 
-    assert_equal <<~OUTPUT, output
+    assert_equal stderr, <<~OUTPUT
       Parsing junit-1.xml
       --- ✍️ Preparing annotation
-      2 testcases found
-      There were no failures/errors 🙌
+    OUTPUT
+
+    assert_equal stdout, <<~OUTPUT
+      Failures: 0
+      Errors: 0
+      Total tests: 2
     OUTPUT
 
     assert_equal 0, status.exitstatus
diff --git a/tests/2-tests-1-failure.output b/tests/2-tests-1-failure.output
new file mode 100644
index 0000000..24f96af
--- /dev/null
+++ b/tests/2-tests-1-failure.output
@@ -0,0 +1,13 @@
+Failures: 0
+Errors: 1
+Total tests: 2
+
+<details>
+<summary><code>Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec</code></summary>
+
+<p>expected: 250 got: 500 (compared using eql?)</p>
+
+<pre><code>First line of failure output
+      Second line of failure output</code></pre>
+
+</details>
diff --git a/tests/command.bats b/tests/command.bats
index 5c2c06c..43396ea 100644
--- a/tests/command.bats
+++ b/tests/command.bats
@@ -1,6 +1,6 @@
 #!/usr/bin/env bats
 
-load "$BATS_PATH/load.bash"
+load "${BATS_PLUGIN_PATH}/load.bash"
 
 # Uncomment to get debug output from each stub
 # export MKTEMP_STUB_DEBUG=/dev/tty
@@ -8,76 +8,106 @@ load "$BATS_PATH/load.bash"
 # export DOCKER_STUB_DEBUG=/dev/tty
 # export DU_STUB_DEBUG=/dev/tty
 
+export artifacts_tmp="tests/tmp/junit-artifacts"
+export annotation_tmp="tests/tmp/junit-annotation"
+export annotation_input="tests/tmp/annotation.input"
+
 @test "runs the annotator and creates the annotation" {
-  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml"
-  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAIL_BUILD_ON_ERROR=false
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_ARTIFACTS="junits/*.xml"
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_FAIL_BUILD_ON_ERROR=false
+
+  stub mktemp \
+    "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \
+    "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'"
+
+  stub buildkite-agent \
+    "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \
+    "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved"
 
-  artifacts_tmp="tests/tmp/$PWD/junit-artifacts"
-  annotation_tmp="tests/tmp/$PWD/junit-annotation"
+  stub docker \
+    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '<details>Failure</details>' && exit 64"
+
+  run "$PWD/hooks/command"
+
+  assert_success
+
+  assert_output --partial "Annotation added with context junit and style error"
+  assert_equal "$(cat "${annotation_input}")" '<details>Failure</details>'
+
+  unstub mktemp
+  unstub buildkite-agent
+  unstub docker
+  rm "${annotation_input}"
+}
+
+@test "can define a special context" {
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_ARTIFACTS="junits/*.xml"
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_CONTEXT="junit_custom_context"
 
   stub mktemp \
-    "-d junit-annotate-plugin-artifacts-tmp.XXXXXXXXXX : mkdir -p $artifacts_tmp; echo $artifacts_tmp" \
-    "-d junit-annotate-plugin-annotation-tmp.XXXXXXXXXX : mkdir -p $annotation_tmp; echo $annotation_tmp"
+    "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \
+    "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'"
 
-  stub buildkite-agent "artifact download junits/*.xml /plugin/tests/tmp//plugin/junit-artifacts : echo Downloaded artifacts" \
-                       "annotate --context junit --style error : echo Annotation added"
+  stub buildkite-agent \
+    "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \
+    "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved"
 
-  stub docker "--log-level error run --rm --volume /plugin/tests/tmp//plugin/junit-artifacts:/junits --volume /plugin/hooks/../ruby:/src --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN= --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT= --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST= ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '<details>Failure</details>' && exit 64"
+  stub docker \
+    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '<details>Failure</details>' && exit 64"
 
   run "$PWD/hooks/command"
 
   assert_success
 
-  assert_output --partial "Annotation added"
+  assert_output --partial "Annotation added with context junit_custom_context"
   
   unstub mktemp
   unstub buildkite-agent
   unstub docker
+  rm "${annotation_input}"
 }
 
-@test "returns an error if fail-build-on-error is true" {
-  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml"
-  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAIL_BUILD_ON_ERROR=true
-
-  artifacts_tmp="tests/tmp/$PWD/junit-artifacts"
-  annotation_tmp="tests/tmp/$PWD/junit-annotation"
+@test "can pass through optional job uuid file pattern" {
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_ARTIFACTS="junits/*.xml"
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_JOB_UUID_FILE_PATTERN="custom_(*)_pattern.xml"
 
   stub mktemp \
-    "-d junit-annotate-plugin-artifacts-tmp.XXXXXXXXXX : mkdir -p $artifacts_tmp; echo $artifacts_tmp" \
-    "-d junit-annotate-plugin-annotation-tmp.XXXXXXXXXX : mkdir -p $annotation_tmp; echo $annotation_tmp"
+    "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \
+    "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'"
 
-  stub buildkite-agent "artifact download junits/*.xml /plugin/tests/tmp//plugin/junit-artifacts : echo Downloaded artifacts" \
-                       "annotate --context junit --style error : echo Annotation added"
+  stub buildkite-agent \
+    "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \
+    "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved"
 
-  stub docker "--log-level error run --rm --volume /plugin/tests/tmp//plugin/junit-artifacts:/junits --volume /plugin/hooks/../ruby:/src --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN= --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT= --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST= ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '<details>Failure</details>' && exit 64"
+  stub docker \
+    "--log-level error run --rm --volume \* --volume \* --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_JOB_UUID_FILE_PATTERN='custom_(*)_pattern.xml' --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '<details>Failure</details>' && exit 64"
 
   run "$PWD/hooks/command"
 
-  assert_failure
+  assert_success
+
+  assert_output --partial "Annotation added"
 
   unstub mktemp
   unstub buildkite-agent
   unstub docker
+  rm "${annotation_input}"
 }
 
-@test "can pass through optional params" {
-  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml"
-  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN="custom_(*)_pattern.xml"
-  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT="file"
-  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAIL_BUILD_ON_ERROR=false
-  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_CONTEXT="junit_custom_context"
-
-  artifacts_tmp="tests/tmp/$PWD/junit-artifacts"
-  annotation_tmp="tests/tmp/$PWD/junit-annotation"
+@test "can pass through optional failure format" {
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_ARTIFACTS="junits/*.xml"
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_FAILURE_FORMAT="file"
 
   stub mktemp \
-    "-d junit-annotate-plugin-artifacts-tmp.XXXXXXXXXX : mkdir -p $artifacts_tmp; echo $artifacts_tmp" \
-    "-d junit-annotate-plugin-annotation-tmp.XXXXXXXXXX : mkdir -p $annotation_tmp; echo $annotation_tmp"
+    "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \
+    "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'"
 
-  stub buildkite-agent "artifact download junits/*.xml /plugin/tests/tmp//plugin/junit-artifacts : echo Downloaded artifacts" \
-                       "annotate --context junit_custom_context --style error : echo Annotation added"
+  stub buildkite-agent \
+    "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \
+    "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved"
 
-  stub docker "--log-level error run --rm --volume /plugin/tests/tmp//plugin/junit-artifacts:/junits --volume /plugin/hooks/../ruby:/src --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN='custom_(*)_pattern.xml' --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT='file' --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST= ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '<details>Failure</details>' && exit 64"
+  stub docker \
+    "--log-level error run --rm --volume \* --volume \* --env \* --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_FAILURE_FORMAT='file' --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '<details>Failure</details>' && exit 64"
 
   run "$PWD/hooks/command"
 
@@ -88,25 +118,51 @@ load "$BATS_PATH/load.bash"
   unstub mktemp
   unstub buildkite-agent
   unstub docker
+  rm "${annotation_input}"
 }
 
 @test "doesn't create annotation unless there's failures" {
-  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml"
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_ARTIFACTS="junits/*.xml"
+
+  stub mktemp \
+    "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \
+    "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'"
 
-  artifacts_tmp="tests/tmp/$PWD/junit-artifacts"
-  annotation_tmp="tests/tmp/$PWD/junit-annotation"
+  stub buildkite-agent \
+    "artifact download \* \* : echo Downloaded artifact \$3 to \$4"
+
+  stub docker \
+    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo No test errors"
+
+  run "$PWD/hooks/command"
+
+  assert_success
+
+  unstub mktemp
+  unstub buildkite-agent
+  unstub docker
+}
+
+@test "creates annotation with no failures but always annotate" {
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_ARTIFACTS="junits/*.xml"
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_ALWAYS_ANNOTATE=1
 
   stub mktemp \
-    "-d junit-annotate-plugin-artifacts-tmp.XXXXXXXXXX : mkdir -p $artifacts_tmp; echo $artifacts_tmp" \
-    "-d junit-annotate-plugin-annotation-tmp.XXXXXXXXXX : mkdir -p $annotation_tmp; echo $annotation_tmp"
+    "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \
+    "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'"
 
-  stub buildkite-agent "artifact download junits/*.xml /plugin/tests/tmp//plugin/junit-artifacts : echo Downloaded artifacts"
+  stub buildkite-agent \
+    "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \
+    "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved"
 
-  stub docker "--log-level error run --rm --volume /plugin/tests/tmp//plugin/junit-artifacts:/junits --volume /plugin/hooks/../ruby:/src --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN= --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT= --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST= ruby:2.7-alpine ruby /src/bin/annotate /junits : echo No test errors"
+  stub docker \
+    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo No test errors"
 
   run "$PWD/hooks/command"
 
   assert_success
+  assert_output --partial "No test errors"
+  assert_output --partial "Will create annotation anyways"
 
   unstub mktemp
   unstub buildkite-agent
@@ -118,55 +174,118 @@ load "$BATS_PATH/load.bash"
 
   assert_failure
 
-  assert_output --partial "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS: unbound variable"
+  assert_output --partial "Missing artifacts configuration for the plugin"
+  refute_output --partial ":junit:"
 }
 
-@test "fails if the annotation is larger than 1MB" {
-  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml"
-
-  artifacts_tmp="tests/tmp/$PWD/junit-artifacts"
-  annotation_tmp="tests/tmp/$PWD/junit-annotation"
+@test "fails if the annotation is larger than 1MB even after summary" {
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_ARTIFACTS="junits/*.xml"
 
   stub mktemp \
-    "-d junit-annotate-plugin-artifacts-tmp.XXXXXXXXXX : mkdir -p $artifacts_tmp; echo $artifacts_tmp" \
-    "-d junit-annotate-plugin-annotation-tmp.XXXXXXXXXX : mkdir -p $annotation_tmp; echo $annotation_tmp"
+    "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \
+    "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'"
 
   # 1KB over the 1MB size limit of annotations
-  stub du "-k /plugin/tests/tmp//plugin/junit-annotation/annotation.md : echo 1025 /plugin/tests/tmp//plugin/junit-annotation/annotation.md"
+  stub du \
+    "-k \* : echo 1025$'\t'\$2" \
+    "-k \* : echo 1025$'\t'\$2"
 
-  stub buildkite-agent "artifact download junits/*.xml /plugin/tests/tmp//plugin/junit-artifacts : echo Downloaded artifacts"
+  stub buildkite-agent \
+    "artifact download \* \* : echo Downloaded artifact \$3 to \$4"
 
-  stub docker "--log-level error run --rm --volume /plugin/tests/tmp//plugin/junit-artifacts:/junits --volume /plugin/hooks/../ruby:/src --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN= --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT= --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST= ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '<details>Failure</details>' && exit 64"
+  stub docker \
+    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '<details>Failure</details>' && exit 64"
 
   run "$PWD/hooks/command"
 
   assert_success
 
   assert_output --partial "Failures too large to annotate"
+  assert_output --partial "failures are too large to create a build annotation"
 
   unstub docker
+  unstub du
   unstub buildkite-agent
+  unstub mktemp
+}
+
+@test "creates summary annotation if original is larger than 1MB" {
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_ARTIFACTS="junits/*.xml"
+
+  stub mktemp \
+    "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \
+    "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'"
+
+  # 1KB over the 1MB size limit of annotations
+  stub du \
+    "-k \* : echo 1025$'\t'\$2" \
+    "-k \* : echo 10$'\t'\$2"
+
+  stub buildkite-agent \
+    "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \
+    "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved"
+
+  stub docker \
+    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64"
+
+  run "$PWD/hooks/command"
+
+  assert_success
+
+  assert_output --partial "Failures too large to annotate"
+  assert_output --partial "using a simplified annotation"
+  assert_equal "5 ${annotation_input}" "$(wc -l "${annotation_input}" | cut -f 1)"
+
+  unstub docker
   unstub du
+  unstub buildkite-agent
   unstub mktemp
+  rm "${annotation_input}"
 }
 
-@test "returns an error if fail-build-on-error is true and annotation is too large" {
-  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml"
-  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAIL_BUILD_ON_ERROR=true
+@test "returns an error if fail-build-on-error is true" {
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_ARTIFACTS="junits/*.xml"
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_FAIL_BUILD_ON_ERROR=true
+
+  stub mktemp \
+    "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \
+    "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'"
 
-  artifacts_tmp="tests/tmp/$PWD/junit-artifacts"
-  annotation_tmp="tests/tmp/$PWD/junit-annotation"
+  stub buildkite-agent \
+    "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \
+    "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved"
+
+  stub docker \
+    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '<details>Failure</details>' && exit 64"
+
+  run "$PWD/hooks/command"
+
+  assert_failure
+
+  unstub mktemp
+  unstub buildkite-agent
+  unstub docker
+  rm "${annotation_input}"
+}
+
+@test "returns an error if fail-build-on-error is true and annotation is too large" {
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_ARTIFACTS="junits/*.xml"
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_FAIL_BUILD_ON_ERROR=true
 
   stub mktemp \
-    "-d junit-annotate-plugin-artifacts-tmp.XXXXXXXXXX : mkdir -p $artifacts_tmp; echo $artifacts_tmp" \
-    "-d junit-annotate-plugin-annotation-tmp.XXXXXXXXXX : mkdir -p $annotation_tmp; echo $annotation_tmp"
+    "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \
+    "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'"
 
   # 1KB over the 1MB size limit of annotations
-  stub du "-k /plugin/tests/tmp//plugin/junit-annotation/annotation.md : echo 1025 /plugin/tests/tmp//plugin/junit-annotation/annotation.md"
+  stub du \
+    "-k \* : echo 1025$'\t'\$2" \
+    "-k \* : echo 1025$'\t'\$2"
   
-  stub buildkite-agent "artifact download junits/*.xml /plugin/tests/tmp//plugin/junit-artifacts : echo Downloaded artifacts"
+  stub buildkite-agent \
+    "artifact download \* \* : echo Downloaded artifact \$3 to \$4"
 
-  stub docker "--log-level error run --rm --volume /plugin/tests/tmp//plugin/junit-artifacts:/junits --volume /plugin/hooks/../ruby:/src --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN= --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT= --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST= ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '<details>Failure</details>' && exit 64"
+  stub docker \
+    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '<details>Failure</details>' && exit 64"
 
   run "$PWD/hooks/command"
 
@@ -175,6 +294,53 @@ load "$BATS_PATH/load.bash"
   assert_output --partial "Failures too large to annotate"
 
   unstub mktemp
+  unstub du
   unstub buildkite-agent
   unstub docker
+}
+
+@test "error bubbles up when ruby code fails with anything but 64" {
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_ARTIFACTS="junits/*.xml"
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_FAIL_BUILD_ON_ERROR=false
+
+  stub mktemp \
+    "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \
+    "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'"
+
+  stub buildkite-agent \
+    "artifact download \* \* : echo Downloaded artifact \$3 to \$4"
+
+  stub docker \
+    "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '<details>Failure</details>' && exit 147"
+
+  run "$PWD/hooks/command"
+
+  assert_failure 147
+
+  assert_output --partial "Error when processing JUnit tests"
+
+  unstub mktemp
+  unstub buildkite-agent
+  unstub docker
+}
+
+@test "error bubbles up when agent download fails" {
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_ARTIFACTS="junits/*.xml"
+  export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_LOCAL_FAIL_BUILD_ON_ERROR=false
+
+  stub mktemp \
+    "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \
+    "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'"
+
+  stub buildkite-agent \
+    "artifact download \* \* : exit 1"
+
+  run "$PWD/hooks/command"
+
+  assert_failure 2
+
+  assert_output --partial "Could not download artifacts"
+
+  unstub mktemp
+  unstub buildkite-agent
 }
\ No newline at end of file