Skip to content

Commit

Permalink
Support Llama 2
Browse files Browse the repository at this point in the history
  • Loading branch information
@borzunov
borzunov committed Jul 19, 2023
1 parent e9d5cdb commit 3cdc76d
Show file tree
Hide file tree
Showing 4 changed files with 69 additions and 53 deletions.
8 changes: 3 additions & 5 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,10 @@ If you develop your own web app, you can use our endpoint at `http://chat.petals

- You don't have to serve all models. If you don't have enough memory, remove some models in [config.py](config.py).

| Model | Embeds in 16-bit | Embeds in 32-bit |
| Model family | Embeds in 16-bit | Embeds in 32-bit |
| --- | --- | --- |
| LLaMA-65B | 1.05 GB | 2.1 GB |
| Guanaco-65B | 1.05 GB | 2.1 GB |
| BLOOM-176B | 7.19 GB | 14.38 GB |
| BLOOMZ-176B | 7.19 GB | 14.38 GB |
| LLaMA 2 (70B, 70B-Chat), LLaMA-65B, Guanaco-65B | 1.05 GB | 2.1 GB |
| BLOOM-176B, BLOOMZ-176B | 7.19 GB | 14.38 GB |

## WebSocket API (`/api/v2/generate`)

Expand Down
6 changes: 4 additions & 2 deletions config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,14 @@ class ModelInfo:


MODELS = [
ModelInfo(repo="meta-llama/Llama-2-70b-hf"),
ModelInfo(repo="meta-llama/Llama-2-70b-chat-hf"),
ModelInfo(repo="enoch/llama-65b-hf"),
ModelInfo(repo="enoch/llama-65b-hf", adapter="timdettmers/guanaco-65b"),
ModelInfo(repo="bigscience/bloom"),
# ModelInfo(repo="bigscience/bloom"),
ModelInfo(repo="bigscience/bloomz"),
]
DEFAULT_MODEL_NAME = "bigscience/bloom"
DEFAULT_MODEL_NAME = "enoch/llama-65b-hf"

INITIAL_PEERS = PUBLIC_INITIAL_PEERS
# Set this to a list of multiaddrs to connect to a private swarm instead of the public one, for example:
Expand Down
65 changes: 36 additions & 29 deletions static/chat.js
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,20 @@
const models = {
"meta-llama/Llama-2-70b-chat-hf": {
modelCard: "https://huggingface.co/meta-llama/Llama-2-70b-chat-hf",
license: "https://bit.ly/llama2-license",
researchOnly: false,
sepToken: "###",
stopToken: "###",
extraStopSequences: ["</s>"],
},
"meta-llama/Llama-2-70b-hf": {
modelCard: "https://huggingface.co/meta-llama/Llama-2-70b-hf",
license: "https://bit.ly/llama2-license",
researchOnly: false,
sepToken: "###",
stopToken: "###",
extraStopSequences: ["</s>"],
},
"timdettmers/guanaco-65b": {
modelCard: "https://huggingface.co/timdettmers/guanaco-65b",
license: "https://huggingface.co/timdettmers/guanaco-65b",
Expand Down Expand Up @@ -32,7 +48,7 @@ const models = {
extraStopSequences: ["\n\nHuman"],
},
};
var curModel = "timdettmers/guanaco-65b";
var curModel = "meta-llama/Llama-2-70b-chat-hf";

const generationParams = {
do_sample: 1,
Expand Down Expand Up @@ -260,6 +276,25 @@ function animateLoading() {
$(() => {
upgradeTextArea();

$('.family-selector label').click(function (e) {
if (!isWaitingForInputs()) {
alert("Can't switch the model while the AI is writing a response. Please refresh the page");
e.preventDefault();
return;
}

const radio = $(`#${$(this).attr("for")}`);
if (radio.is(":checked")) {
setTimeout(() => $('.human-replica textarea').focus(), 10);
return;
}

const curFamily = radio.attr("value");
$('.model-selector').hide();
const firstLabel = $(`.model-selector[data-family=${curFamily}]`).show().children('label:first');
firstLabel.click();
firstLabel.trigger('click');
});
$('.model-selector label').click(function (e) {
if (!isWaitingForInputs()) {
alert("Can't switch the model while the AI is writing a response. Please refresh the page");
Expand All @@ -283,34 +318,6 @@ $(() => {
$('.license-link').attr('href', models[curModel].license);
setTimeout(() => $('.human-replica textarea').focus(), 10);
});
$('.regime-selector label').click(function (e) {
if (!isWaitingForInputs()) {
alert("Can't switch the regime while the AI is writing a response. Please refresh the page");
e.preventDefault();
return;
}

$('.dialogue').empty();
if ($(this).attr("for") === "regime-chatbot") {
location.reload();
return;
}

curRegime = Regime.FEW_SHOT;
resetSession();
appendTextArea();

const textarea = $('.human-replica textarea');
textarea.val(
'Input: A cat sat on a mat.\n\n' +
'Output: Un gato se sentó en una estera.\n\n' +
'Input: A brown fox jumps over the lazy dog.\n\n' +
'Output: Un zorro marrón salta sobre el perro perezoso.\n\n' +
'Input: Who is the president of the United States?'
);
textarea[0].style.height = textarea[0].scrollHeight + "px";
setTimeout(() => textarea.focus(), 10);
});
$('.retry-link').click(e => {
e.preventDefault();
retry();
Expand Down
43 changes: 26 additions & 17 deletions static/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,37 +35,46 @@ <h1 class="display-4 fw-bold d-md-inline-block justify-content-center" style="fo
<div class="welcome mb-4">
<div>
<b>Welcome!</b> This is a demo app running
<a target="_blank" class="model-name" href="https://huggingface.co/timdettmers/guanaco-65b">Guanaco-65B</a>
<a target="_blank" class="model-name" href="https://huggingface.co/meta-llama/Llama-2-70b-chat-hf">LLaMA 2 (70B-chat)</a>
over the <a target="_blank" href="https://petals.ml">Petals</a> network.
<span class="research-only">This model is available only for non-commercial research purposes.</span>
Please follow the model's
<a target="_blank" class="license-link" href="https://huggingface.co/timdettmers/guanaco-65b">terms of use</a>
<a target="_blank" class="license-link" href="https://bit.ly/llama2-license">terms of use</a>
and do not enter sensitive data.
The chat history is recorded.
</div>
<form id="settings">
<div class="mt-2">
<label class="group-label">Family:</label>
<div class="btn-group family-selector" role="group">
<input type="radio" class="btn-check" name="family" value="llama-2" id="family-llama-2" checked>
<label class="btn btn-outline-primary" for="family-llama-2">LLaMA 2</label>
<input type="radio" class="btn-check" name="family" value="llama" id="family-llama">
<label class="btn btn-outline-primary" for="family-llama">LLaMA</label>
<input type="radio" class="btn-check" name="family" value="bloom" id="family-bloom">
<label class="btn btn-outline-primary" for="family-bloom">BLOOM</label>
</div>
</div>
<div class="mt-2">
<label class="group-label">Model:</label>
<div class="btn-group model-selector" role="group">
<input type="radio" class="btn-check" name="model" value="timdettmers/guanaco-65b" id="model-guanaco-65b" checked>
<div class="model-selector btn-group" role="group" data-family="llama-2">
<input type="radio" class="btn-check" name="model" value="meta-llama/Llama-2-70b-chat-hf" id="meta-llama-2-70b-chat-hf" checked>
<label class="btn btn-outline-primary" for="meta-llama-2-70b-chat-hf">LLaMA 2 (70B-chat)</label>
<input type="radio" class="btn-check" name="model" value="meta-llama/Llama-2-70b-hf" id="meta-llama-2-70b-hf">
<label class="btn btn-outline-primary" for="meta-llama-2-70b-hf">LLaMA 2 (70B)</label>
</div>
<div class="model-selector btn-group" role="group" data-family="llama" style="display: none;">
<input type="radio" class="btn-check" name="model" value="timdettmers/guanaco-65b" id="model-guanaco-65b">
<label class="btn btn-outline-primary" for="model-guanaco-65b">Guanaco-65B</label>
<input type="radio" class="btn-check" name="model" value="enoch/llama-65b-hf" id="model-llama-65b-hf">
<label class="btn btn-outline-primary" for="model-llama-65b-hf">LLaMA-65B</label>
<input type="radio" class="btn-check" name="model" value="bigscience/bloom" id="model-bloom">
<label class="btn btn-outline-primary" for="model-bloom">BLOOM</label>
</div>
<div class="model-selector btn-group" role="group" data-family="bloom" style="display: none;">
<!-- <input type="radio" class="btn-check" name="model" value="bigscience/bloom" id="model-bloom">
<label class="btn btn-outline-primary" for="model-bloomz">BLOOM-176B</label> -->
<input type="radio" class="btn-check" name="model" value="bigscience/bloomz" id="model-bloomz">
<label class="btn btn-outline-primary" for="model-bloomz">BLOOMZ</label>
<label class="btn btn-outline-primary" for="model-bloomz">BLOOMZ-176B</label>
</div>
</div>
<!-- <div class="mt-2">
<label class="group-label">Regime:</label>
<div class="btn-group regime-selector" role="group">
<input type="radio" class="btn-check" name="regime" value="chatbot" id="regime-chatbot" checked>
<label class="btn btn-outline-primary" for="regime-chatbot">Chatbot</label>
<input type="radio" class="btn-check" name="regime" value="few-shot" id="regime-few-shot">
<label class="btn btn-outline-primary" for="regime-few-shot">Few-shot</label>
</div>
</div> -->
</form>
</div>
<div class="dialogue">
Expand Down

0 comments on commit 3cdc76d

Please sign in to comment.