From b01c56d3dbf9b40ee958efb7369fb5a7b16ac388 Mon Sep 17 00:00:00 2001 From: Chan Jun Shern Date: Wed, 22 May 2024 23:53:27 +0100 Subject: [PATCH 1/2] Remove problematic sample from schelling point dataset --- evals/registry/data/schelling_point/mix.jsonl | 4 ++-- evals/registry/data/schelling_point/owt_5.jsonl | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/evals/registry/data/schelling_point/mix.jsonl b/evals/registry/data/schelling_point/mix.jsonl index f1f60c2c42..d7b28c1d83 100644 --- a/evals/registry/data/schelling_point/mix.jsonl +++ b/evals/registry/data/schelling_point/mix.jsonl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5593a9f39fe34336b91008f7f7fddf9ddd56e3fda45f973d125ff2bfd0ac5cf7 -size 16129977 +oid sha256:a7d2c46729473fbd01dca750912d334487a6809c4dc8cfa58e8c7dc34f75fc1f +size 16074461 diff --git a/evals/registry/data/schelling_point/owt_5.jsonl b/evals/registry/data/schelling_point/owt_5.jsonl index 0a3f40c2db..6fa67ccf70 100644 --- a/evals/registry/data/schelling_point/owt_5.jsonl +++ b/evals/registry/data/schelling_point/owt_5.jsonl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:095ceeb4ac1248bcaac4952056c6ccd5872410fc6a8b55d46244d57f27cfbb8b -size 7620781 +oid sha256:929794988b95c964c39fe2aad0ff043908d904ce4a92a79ac23647a7e187fe4e +size 7565265 From a506f0c171153f86caebbe00023f02648d2326a1 Mon Sep 17 00:00:00 2001 From: Chan Jun Shern Date: Wed, 22 May 2024 23:54:28 +0100 Subject: [PATCH 2/2] Add NotImplementedError if user tries to use a solver --- evals/elsuite/schelling_point/eval.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/evals/elsuite/schelling_point/eval.py b/evals/elsuite/schelling_point/eval.py index 46d5371af1..78f6be68c6 100644 --- a/evals/elsuite/schelling_point/eval.py +++ b/evals/elsuite/schelling_point/eval.py @@ -9,6 +9,7 @@ from evals.elsuite.schelling_point.prompts import sys_prompts_ci, sys_prompts_no_ci from evals.elsuite.schelling_point.utils import get_response from evals.eval import Eval +from evals.solvers.solver import Solver class SchellingPoint(Eval): @@ -22,6 +23,9 @@ def __init__( *args, **kwargs, ): + if any([isinstance(completion_fn, Solver) for completion_fn in completion_fns]): + raise NotImplementedError("SchellingPoint does not currently support Solvers") + super().__init__(completion_fns, *args, **kwargs) random.seed(seed) @@ -34,7 +38,6 @@ def __init__( self.temperature = temperature - self.completion_fns = completion_fns if len(self.completion_fns) == 1: self.completion_fns = self.completion_fns * n_copies assert len(self.completion_fns) == n_copies, "Must provide n_copies completion_fns"