count tokens

book_maker/utils.py

@@ -117,3 +117,25 @@
     "sinhalese": "si",
     "castilian": "es",
 }
+
+import tiktoken
+
+def num_tokens_from_messages(messages, model="gpt-3.5-turbo"):
+    """Returns the number of tokens used by a list of messages."""
+    try:
+        encoding = tiktoken.encoding_for_model(model)
+    except KeyError:
+        encoding = tiktoken.get_encoding("cl100k_base")
+    if model == "gpt-3.5-turbo":  # note: future models may deviate from this
+        num_tokens = 0
+        for message in messages:
+            num_tokens += 4  # every message follows <im_start>{role/name}\n{content}<im_end>\n
+            for key, value in message.items():
+                num_tokens += len(encoding.encode(value))
+                if key == "name":  # if there's a name, the role is omitted
+                    num_tokens += -1  # role is always required and always 1 token
+        num_tokens += 2  # every reply is primed with <im_start>assistant
+        return num_tokens
+    else:
+        raise NotImplementedError(f"""num_tokens_from_messages() is not presently implemented for model {model}.
+See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens.""")

requirements.txt

@@ -1,5 +1,6 @@
 bs4
 openai
+tiktoken
 requests
 ebooklib
 rich