AnthropicSolver (#1498)

This PR contributes an `AnthropicSolver` class, a solver for using
models served by the [Anthropic Claude
API](https://docs.anthropic.com/claude/docs/intro-to-claude), such as
claude 3.

Besides basic functionality, the solver provides the following features

- [x] Handles backoff
- [x] Handles CoT and other solvers with non-alternating roles
- [x] token usage estimate

Notes:

- logit biasing not supported by the anthropic API
- checking for context length limits not supported; anthropic have not
released a tokenizer yet (like
[tiktoken](https://github.com/openai/tiktoken) from openai)
- supports chat models only. if anthropic releases base models at some
point, we will address that when it arises

evals/registry/solvers/anthropic.yaml

@@ -0,0 +1,125 @@
+# ------------------
+# claude-3-opus-20240229
+# ------------------
+
+generation/direct/claude-3-opus-20240229:
+  class: evals.solvers.providers.anthropic.anthropic_solver:AnthropicSolver
+  args:
+    model_name: claude-3-opus-20240229
+
+generation/cot/claude-3-opus-20240229:
+  class: evals.solvers.nested.cot_solver:CoTSolver
+  args:
+    cot_solver:
+      class: evals.solvers.providers.anthropic.anthropic_solver:AnthropicSolver
+      args:
+        model_name: claude-3-opus-20240229
+    extract_solver:
+      class: evals.solvers.providers.anthropic.anthropic_solver:AnthropicSolver
+      args:
+        model_name: claude-3-opus-20240229
+
+# ------------------
+# claude-3-sonnet-20240229
+# ------------------
+
+generation/direct/claude-3-sonnet-20240229:
+  class: evals.solvers.providers.anthropic.anthropic_solver:AnthropicSolver
+  args:
+    model_name: claude-3-sonnet-20240229
+
+generation/cot/claude-3-sonnet-20240229:
+  class: evals.solvers.nested.cot_solver:CoTSolver
+  args:
+    cot_solver:
+      class: evals.solvers.providers.anthropic.anthropic_solver:AnthropicSolver
+      args:
+        model_name: claude-3-sonnet-20240229
+    extract_solver:
+      class: evals.solvers.providers.anthropic.anthropic_solver:AnthropicSolver
+      args:
+        model_name: claude-3-sonnet-20240229
+
+# ------------------
+# claude-3-haiku-20240307
+# ------------------
+
+generation/direct/claude-3-haiku-20240307:
+  class: evals.solvers.providers.anthropic.anthropic_solver:AnthropicSolver
+  args:
+    model_name: claude-3-haiku-20240307
+
+generation/cot/claude-3-haiku-20240307:
+  class: evals.solvers.nested.cot_solver:CoTSolver
+  args:
+    cot_solver:
+      class: evals.solvers.providers.anthropic.anthropic_solver:AnthropicSolver
+      args:
+        model_name: claude-3-haiku-20240307
+    extract_solver:
+      class: evals.solvers.providers.anthropic.anthropic_solver:AnthropicSolver
+      args:
+        model_name: claude-3-haiku-20240307
+
+# ------------------
+# claude-2.1
+# ------------------
+
+generation/direct/claude-2.1:
+  class: evals.solvers.providers.anthropic.anthropic_solver:AnthropicSolver
+  args:
+    model_name: claude-2.1
+
+generation/cot/claude-2.1:
+  class: evals.solvers.nested.cot_solver:CoTSolver
+  args:
+    cot_solver:
+      class: evals.solvers.providers.anthropic.anthropic_solver:AnthropicSolver
+      args:
+        model_name: claude-2.1
+    extract_solver:
+      class: evals.solvers.providers.anthropic.anthropic_solver:AnthropicSolver
+      args:
+        model_name: claude-2.1
+
+# ------------------
+# claude-2.0
+# ------------------
+
+generation/direct/claude-2.0:
+  class: evals.solvers.providers.anthropic.anthropic_solver:AnthropicSolver
+  args:
+    model_name: claude-2.0
+
+generation/cot/claude-2.0:
+  class: evals.solvers.nested.cot_solver:CoTSolver
+  args:
+    cot_solver:
+      class: evals.solvers.providers.anthropic.anthropic_solver:AnthropicSolver
+      args:
+        model_name: claude-2.0
+    extract_solver:
+      class: evals.solvers.providers.anthropic.anthropic_solver:AnthropicSolver
+      args:
+        model_name: claude-2.0
+
+# ------------------
+# claude-instant-1.2
+# ------------------
+
+generation/direct/claude-instant-1.2:
+  class: evals.solvers.providers.anthropic.anthropic_solver:AnthropicSolver
+  args:
+    model_name: claude-instant-1.2
+
+generation/cot/claude-instant-1.2:
+  class: evals.solvers.nested.cot_solver:CoTSolver
+  args:
+    cot_solver:
+      class: evals.solvers.providers.anthropic.anthropic_solver:AnthropicSolver
+      args:
+        model_name: claude-instant-1.2
+    extract_solver:
+      class: evals.solvers.providers.anthropic.anthropic_solver:AnthropicSolver
+      args:
+        model_name: claude-instant-1.2

evals/solvers/providers/anthropic/anthropic_solver.py

@@ -0,0 +1,148 @@
+from typing import Any, Optional, Union
+
+from evals.solvers.solver import Solver, SolverResult
+from evals.task_state import TaskState, Message
+from evals.record import record_sampling
+from evals.utils.api_utils import request_with_timeout
+
+import anthropic
+from anthropic import Anthropic
+from anthropic.types import ContentBlock, MessageParam, Usage
+import backoff
+
+oai_to_anthropic_role = {
+    "system": "user",
+    "user": "user",
+    "assistant": "assistant",
+}
+
+
+class AnthropicSolver(Solver):
+    """
+    A solver class that uses the Anthropic API for textual chat-based tasks.
+    """
+
+    def __init__(
+        self,
+        model_name: str,
+        max_tokens: int = 512,
+        postprocessors: list[str] = [],
+        extra_options: Optional[dict] = {},
+        registry: Any = None,
+    ):
+        super().__init__(postprocessors=postprocessors)
+        # https://docs.anthropic.com/claude/docs/models-overview#model-comparison
+        self.model_name = model_name
+        self.max_tokens = max_tokens
+        self.extra_options = extra_options
+
+    def _solve(self, task_state: TaskState, **kwargs) -> SolverResult:
+        """
+        Solve the task using the Anthropic API
+        """
+        orig_msgs = task_state.messages
+        anth_msgs = self._convert_msgs_to_anthropic_format(task_state.messages)
+
+        # TODO: handle context length limit; possible once anthropic tokenizer is available
+
+        # calls client.messages.create, but is wrapped with backoff retrying decorator
+        response = anthropic_create_retrying(
+            client=Anthropic(max_retries=0),  # we take care of retries ourselves
+            model=self.model_name,
+            system=task_state.task_description,
+            messages=anth_msgs,
+            max_tokens=self.max_tokens,  # required kwarg for messages.create
+            **{**kwargs, **self.extra_options},
+        )
+        solver_result = SolverResult(
+            output=response.content[0].text, raw_completion_result=response.content
+        )
+
+        # for logging purposes: prepend the task desc to the orig msgs as a system message
+        orig_msgs.insert(
+            0, Message(role="system", content=task_state.task_description).to_dict()
+        )
+        record_sampling(
+            prompt=orig_msgs,  # original message format, supported by our logviz
+            sampled=[solver_result.output],
+            model=self.model_name,
+            usage=anth_to_openai_usage(response.usage),
+        )
+        return solver_result
+
+    @property
+    def name(self) -> str:
+        return self.model_name
+
+    @property
+    def model_version(self) -> Union[str, dict]:
+        """
+        For the moment, Anthropic does not use aliases,
+        so model_version is the same as model_name.
+        """
+        return self.model_name
+
+    @staticmethod
+    def _convert_msgs_to_anthropic_format(msgs: list[Message]) -> list[MessageParam]:
+        """
+        Anthropic API requires that the message list has
+        - Roles as 'user' or 'assistant'
+        - Alternating 'user' and 'assistant' messages
+
+        Note: the top-level system prompt is handled separately and should not be
+        included in the messages list.
+        """
+        # enforce valid roles; convert to Anthropic message type
+        anth_msgs = [
+            MessageParam(
+                role=oai_to_anthropic_role[msg.role],
+                content=[ContentBlock(text=msg.content, type="text")],
+            )
+            for msg in msgs
+        ]
+        # enforce alternating roles by merging consecutive messages with the same role
+        # e.g. [user1, user2, assistant1, user3] -> [user12, assistant1, user3]
+        alt_msgs = []
+        for msg in anth_msgs:
+            if len(alt_msgs) > 0 and msg["role"] == alt_msgs[-1]["role"]:
+                # Merge consecutive messages from the same role
+                alt_msgs[-1]["content"].extend(msg["content"])
+            else:
+                alt_msgs.append(msg)
+
+        return alt_msgs
+
+
+@backoff.on_exception(
+    wait_gen=backoff.expo,
+    exception=(
+        anthropic.RateLimitError,
+        anthropic.APIConnectionError,
+        anthropic.APITimeoutError,
+        anthropic.InternalServerError,
+    ),
+    max_value=60,
+    factor=1.5,
+)
+def anthropic_create_retrying(client: Anthropic, *args, **kwargs):
+    """
+    Helper function for creating a backoff-retry enabled message request.
+    `args` and `kwargs` match what is accepted by `client.messages.create`.
+    """
+    result = request_with_timeout(client.messages.create, *args, **kwargs)
+    if "error" in result:
+        raise Exception(result["error"])
+    return result
+
+
+def anth_to_openai_usage(anth_usage: Usage) -> dict:
+    """
+    Processes anthropic Usage object into dict with keys
+    that match the OpenAI Usage dict, for logging purposes.
+    """
+    # TODO: make this format of dict a dataclass type to be reused througout lib?
+    return {
+        "completion_tokens": anth_usage.output_tokens,
+        "prompt_tokens": anth_usage.input_tokens,
+        "total_tokens": anth_usage.input_tokens + anth_usage.output_tokens,
+    }