update YAML

openai · Oct 17, 2023 · 23c9a46 · 23c9a46
1 parent f10b63b
commit 23c9a46
Showing 1 changed file with 17 additions and 3 deletions.
diff --git a/evals/registry/evals/hanabi_clues.yaml b/evals/registry/evals/hanabi_clues.yaml
@@ -4,11 +4,25 @@ hanabi_clues:
   metrics:
   - accuracy
   - f1_score
+
 hanabi_clues-list_all_valid_color_and_number.full-v0:
   args:
     samples_jsonl: hanabi_clues/hanabi_hands_unclued.jsonl
-  class: evals.elsuite.basic.fuzzy_match:FuzzyMatch
+    use_exclusions: False # does not mark a problem incorrect for the inclusion of an invalid color or number clue
+  class: evals.elsuite.basic.includes:Includes
 hanabi_clues-list_all_valid_color_and_number.small-v0:
   args:
-    samples_jsonl: hanabi_clues/hanabi_hands_unclued.jsonl
-  class: evals.elsuite.basic.fuzzy_match:FuzzyMatch
+    samples_jsonl: hanabi_clues/hanabi_hands_unclued_small.jsonl
+    use_exclusions: False # does not mark a problem incorrect for the inclusion of an invalid color or number clue
+  class: evals.elsuite.basic.includes:Includes
+
+hanabi_clues-list_all_valid_color_and_number-use_exclusions.full-v0:
+  args:
+    samples_jsonl: hanabi_clues/hanabi_hands_unclued_with_exclusions.jsonl
+    use_exclusions: True # marks a problem incorrect for the inclusion of an invalid color or number clue
+  class: evals.elsuite.basic.includes:Includes
+hanabi_clues-list_all_valid_color_and_number-use_exclusions.small-v0:
+  args:
+    samples_jsonl: hanabi_clues/hanabi_hands_unclued_with_exclusions_small.jsonl
+    use_exclusions: True # marks a problem incorrect for the inclusion of an invalid color or number clue
+  class: evals.elsuite.basic.includes:Includes