Skip to content

Commit

Permalink
enable prompt caching
Browse files Browse the repository at this point in the history
  • Loading branch information
@imolorhe
imolorhe committed Nov 9, 2024
1 parent 1630ab9 commit da3eddf
Show file tree
Hide file tree
Showing 3 changed files with 94 additions and 58 deletions.
6 changes: 3 additions & 3 deletions packages/altair-api/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
"@altairgraphql/api-utils": "^8.0.1",
"@altairgraphql/db": "^8.0.1",
"@altairgraphql/emails": "^8.0.1",
"@langchain/anthropic": "^0.2.4",
"@langchain/community": "^0.2.19",
"@langchain/openai": "^0.2.2",
"@langchain/anthropic": "^0.3.7",
"@langchain/community": "^0.3.12",
"@langchain/openai": "^0.3.12",
"@nestjs/common": "^10.3.10",
"@nestjs/config": "^3.2.3",
"@nestjs/core": "^10.3.10",
Expand Down
15 changes: 14 additions & 1 deletion packages/altair-api/src/ai/ai.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,15 @@ export class AiService {
dedent`Now, please answer the following user question:`,
];
const promptTemplate = ChatPromptTemplate.fromMessages([
new SystemMessage(systemMessageParts.join('\n\n')),
new SystemMessage({
content: [
{
text: systemMessageParts.join('\n\n'),
type: 'text',
cache_control: { type: 'ephemeral' },
},
],
}),
...previousMessages.map((m) => {
if (m.role === AiChatRole.USER) {
return new HumanMessage(m.message);
Expand Down Expand Up @@ -311,6 +319,11 @@ export class AiService {
apiKey: this.configService.get('ai.anthropic.apiKey', { infer: true }),
model: this.configService.get('ai.anthropic.model', { infer: true }),
maxTokens: responseMaxTokens,
clientOptions: {
defaultHeaders: {
'anthropic-beta': 'prompt-caching-2024-07-31',
},
},
});
}
}
Expand Down
131 changes: 77 additions & 54 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -541,10 +541,10 @@
"@ant-design/colors" "^5.0.0"
tslib "^2.0.0"

"@anthropic-ai/sdk@^0.22.0":
version "0.22.0"
resolved "https://registry.yarnpkg.com/@anthropic-ai/sdk/-/sdk-0.22.0.tgz#548e4218d9810fd494e595d4e57cb2d46d301a1a"
integrity sha512-dv4BCC6FZJw3w66WNLsHlUFjhu19fS1L/5jMPApwhZLa/Oy1j0A2i3RypmDtHEPp4Wwg3aZkSHksp7VzYWjzmw==
"@anthropic-ai/sdk@^0.27.3":
version "0.27.3"
resolved "https://registry.yarnpkg.com/@anthropic-ai/sdk/-/sdk-0.27.3.tgz#592cdd873c85ffab9589ae6f2e250cbf150e1475"
integrity sha512-IjLt0gd3L4jlOfilxVXTifn42FnVffMgDC04RJK1KDZpmkBWLv0XC92MVVmkxrFZNS/7l3xWgP/I3nqtX1sQHw==
dependencies:
"@types/node" "^18.11.18"
"@types/node-fetch" "^2.6.4"
Expand All @@ -553,7 +553,6 @@
form-data-encoder "1.7.2"
formdata-node "^4.3.2"
node-fetch "^2.6.7"
web-streams-polyfill "^3.2.1"

"@apollo/[email protected]":
version "3.3.20"
Expand Down Expand Up @@ -4778,35 +4777,33 @@
jsbi "^4.3.0"
tslib "^2.4.1"

"@langchain/anthropic@^0.2.4":
version "0.2.4"
resolved "https://registry.yarnpkg.com/@langchain/anthropic/-/anthropic-0.2.4.tgz#80150d4aaa3f07b4857fd5c4ade0de5bb7bd3bed"
integrity sha512-BbWKpxvF1uoix1c4k/jPqAses6Coy8q81CmT1XHKdOdFQ0t0PSsZPYf0K85aWscB9Wo7S2Nb2Dhfc+oGZjPIQQ==
"@langchain/anthropic@^0.3.7":
version "0.3.7"
resolved "https://registry.yarnpkg.com/@langchain/anthropic/-/anthropic-0.3.7.tgz#4fa16b56e931c0ae48144c4f99cd238d4ed32b66"
integrity sha512-MjV7BNPalnG3S6PqXYHRtv3nEML1fFHl9OsqjT5KCPcULxJImnIZrJX5qMTnezM5A+Q6KOZt3e07x7aYCmU3Sg==
dependencies:
"@anthropic-ai/sdk" "^0.22.0"
"@langchain/core" ">=0.2.16 <0.3.0"
fast-xml-parser "^4.3.5"
"@anthropic-ai/sdk" "^0.27.3"
fast-xml-parser "^4.4.1"
zod "^3.22.4"
zod-to-json-schema "^3.22.4"

"@langchain/community@^0.2.19":
version "0.2.19"
resolved "https://registry.yarnpkg.com/@langchain/community/-/community-0.2.19.tgz#ca1fb64e57f94216d6d05aa9362950d4c5473bc3"
integrity sha512-NKUOFW7ykY+WcnxEV6MZJj1hKncogdloBGDsk5zfW/FkZtQQpSHTgA8bgAT7X4Bnr5+Cv1fLkiDtVs/yKI4/Ow==
"@langchain/community@^0.3.12":
version "0.3.12"
resolved "https://registry.yarnpkg.com/@langchain/community/-/community-0.3.12.tgz#6d19cb7ae00c9e3b5eeababc93271145c59f5f8d"
integrity sha512-nGj/wyuARf/54JM0ey+MUP2nbuhe9fmdl+pZPCtescveDJL4b+2eaiaNMEYWDgpakSDTymh90wGTYgs9XLzIBg==
dependencies:
"@langchain/core" ">=0.2.16 <0.3.0"
"@langchain/openai" "~0.1.0"
"@langchain/openai" ">=0.2.0 <0.4.0"
binary-extensions "^2.2.0"
expr-eval "^2.0.2"
flat "^5.0.2"
js-yaml "^4.1.0"
langchain "0.2.3"
langsmith "~0.1.30"
langchain ">=0.2.3 <0.3.0 || >=0.3.4 <0.4.0"
langsmith "^0.2.0"
uuid "^10.0.0"
zod "^3.22.3"
zod-to-json-schema "^3.22.5"

"@langchain/core@>0.1.56 <0.3.0", "@langchain/core@>0.2.0 <0.3.0", "@langchain/core@>=0.2.11 <0.3.0", "@langchain/core@>=0.2.16 <0.3.0", "@langchain/core@>=0.2.5 <0.3.0", "@langchain/core@~0.2.0":
"@langchain/core@>0.2.0 <0.3.0", "@langchain/core@>=0.2.11 <0.3.0", "@langchain/core@>=0.2.16 <0.3.0":
version "0.2.16"
resolved "https://registry.yarnpkg.com/@langchain/core/-/core-0.2.16.tgz#0700a7def44c613ef775d351a2e6428e09cfbfda"
integrity sha512-mPmQi0ecJ81QwhvUQX4cwGVAqsM30ly3ygIlWoeUwDOXv9UW/IB2LAq8KKoVYIHTyEsIWJiyMP9Sv3e0xwjV8g==
Expand All @@ -4824,7 +4821,7 @@
zod "^3.22.4"
zod-to-json-schema "^3.22.3"

"@langchain/openai@>=0.1.0 <0.3.0", "@langchain/openai@^0.2.2":
"@langchain/openai@>=0.1.0 <0.3.0":
version "0.2.2"
resolved "https://registry.yarnpkg.com/@langchain/openai/-/openai-0.2.2.tgz#dec162af357477237f376a1b9a282b9c999c3ea2"
integrity sha512-1V7GZbRf8srdgfHV8C3W63BlYMf6/hsjTKh5ozZQaQDQ6bWMJwDjauvISvSVA+A0tqLAzf017C4+QVvy1kxjsw==
Expand All @@ -4835,27 +4832,22 @@
zod "^3.22.4"
zod-to-json-schema "^3.22.3"

"@langchain/openai@~0.0.28":
version "0.0.34"
resolved "https://registry.yarnpkg.com/@langchain/openai/-/openai-0.0.34.tgz#36c9bca0721ab9f7e5d40927e7c0429cacbd5b56"
integrity sha512-M+CW4oXle5fdoz2T2SwdOef8pl3/1XmUx1vjn2mXUVM/128aO0l23FMF0SNBsAbRV6P+p/TuzjodchJbi0Ht/A==
"@langchain/openai@>=0.1.0 <0.4.0", "@langchain/openai@>=0.2.0 <0.4.0", "@langchain/openai@^0.3.12":
version "0.3.12"
resolved "https://registry.yarnpkg.com/@langchain/openai/-/openai-0.3.12.tgz#b92c2934c824d9a564b332ff4785cb49628ae673"
integrity sha512-z8y0zdKovjY6bB0D3D+K99bPAN+wFRQcrWDfHfX7yVmYwq1O8GOIzQgCBth0vbhbzeuaJ4dz3+oqN/4Q2PCpHg==
dependencies:
"@langchain/core" ">0.1.56 <0.3.0"
js-tiktoken "^1.0.12"
openai "^4.41.1"
openai "^4.71.0"
zod "^3.22.4"
zod-to-json-schema "^3.22.3"

"@langchain/openai@~0.1.0":
version "0.1.3"
resolved "https://registry.yarnpkg.com/@langchain/openai/-/openai-0.1.3.tgz#6eb0994e970d85ffa9aaeafb94449024ccf6ca63"
integrity sha512-riv/JC9x2A8b7GcHu8sx+mlZJ8KAwSSi231IPTlcciYnKozmrQ5H0vrtiD31fxiDbaRsk7tyCpkSBIOQEo7CyQ==
"@langchain/textsplitters@>=0.0.0 <0.2.0":
version "0.1.0"
resolved "https://registry.yarnpkg.com/@langchain/textsplitters/-/textsplitters-0.1.0.tgz#f37620992192df09ecda3dfbd545b36a6bcbae46"
integrity sha512-djI4uw9rlkAb5iMhtLED+xJebDdAG935AdP4eRTB02R7OB/act55Bj9wsskhZsvuyQRpO4O1wQOp85s6T6GWmw==
dependencies:
"@langchain/core" ">=0.2.5 <0.3.0"
js-tiktoken "^1.0.12"
openai "^4.49.1"
zod "^3.22.4"
zod-to-json-schema "^3.22.3"

"@langchain/textsplitters@~0.0.0":
version "0.0.3"
Expand Down Expand Up @@ -9033,6 +9025,11 @@
resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.10.tgz#04ffa7f406ab628f7f7e97ca23e290cd8ab15efc"
integrity sha512-IfYcSBWE3hLpBg8+X2SEa8LVkJdJEkT2Ese2aaLs3ptGdVtABxndrMaxuFlQ1qdFf9Q5rDvDpxI3WwgvKFAsQA==

"@types/uuid@^10.0.0":
version "10.0.0"
resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-10.0.0.tgz#e9c07fe50da0f53dc24970cca94d619ff03f6f6d"
integrity sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==

"@types/uuid@^9.0.1":
version "9.0.8"
resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-9.0.8.tgz#7545ba4fc3c003d6c756f651f3bf163d8f0f29ba"
Expand Down Expand Up @@ -18554,10 +18551,10 @@ fast-uri@^3.0.0, fast-uri@^3.0.1:
resolved "https://registry.yarnpkg.com/fast-uri/-/fast-uri-3.0.2.tgz#d78b298cf70fd3b752fd951175a3da6a7b48f024"
integrity sha512-GR6f0hD7XXyNJa25Tb9BuIdN0tdr+0BMi6/CJPH3wJO1JjNG3n/VsSw38AwRdKZABm8lGbPfakLRkYzx2V9row==

fast-xml-parser@^4.3.5:
version "4.4.0"
resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.4.0.tgz#341cc98de71e9ba9e651a67f41f1752d1441a501"
integrity sha512-kLY3jFlwIYwBNDojclKsNAC12sfD6NwW74QB2CoNGPvtVxjliYehVunB3HYyNi+n4Tt1dAcgwYvmKF/Z18flqg==
fast-xml-parser@^4.4.1:
version "4.5.0"
resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.5.0.tgz#2882b7d01a6825dfdf909638f2de0256351def37"
integrity sha512-/PlTQCI96+fZMAOLMZK4CWG1ItCbfZ/0jx7UIJFChPNrx7tcEgerUgWbeieCM9MfHInUDyK8DWYZ+YrywDJuTg==
dependencies:
strnum "^1.0.5"

Expand Down Expand Up @@ -23815,24 +23812,20 @@ ky@^0.33.2:
resolved "https://registry.yarnpkg.com/ky/-/ky-0.33.2.tgz#76ad8f0403d1e6531f1a5931c79716ed420492b5"
integrity sha512-f6oS2rKUcPu5FzdqCDbFpmzis/JlqFZw8uIHm/jf8Kc3vtnW+VDhuashOAKyBZv8bFiZFZUMNxTC0JtahEvujA==

[email protected]:
version "0.2.3"
resolved "https://registry.yarnpkg.com/langchain/-/langchain-0.2.3.tgz#c14bb05cf871b21bd63b84b3ab89580b1d62539f"
integrity sha512-T9xR7zd+Nj0oXy6WoYKmZLy0DlQiDLFPGYWdOXDxy+AvqlujoPdVQgDSpdqiOHvAjezrByAoKxoHCz5XMwTP/Q==
"langchain@>=0.2.3 <0.3.0 || >=0.3.4 <0.4.0":
version "0.3.5"
resolved "https://registry.yarnpkg.com/langchain/-/langchain-0.3.5.tgz#87b282454bc215b12b920d4dd5e35ed58030bad1"
integrity sha512-Gq0xC45Sq6nszS8kQG9suCrmBsuXH0INMmiF7D2TwPb6mtG35Jiq4grCk9ykpwPsarTHdty3SzUbII/FqiYSSw==
dependencies:
"@langchain/core" "~0.2.0"
"@langchain/openai" "~0.0.28"
"@langchain/textsplitters" "~0.0.0"
binary-extensions "^2.2.0"
"@langchain/openai" ">=0.1.0 <0.4.0"
"@langchain/textsplitters" ">=0.0.0 <0.2.0"
js-tiktoken "^1.0.12"
js-yaml "^4.1.0"
jsonpointer "^5.0.1"
langchainhub "~0.0.8"
langsmith "~0.1.7"
ml-distance "^4.0.0"
langsmith "^0.2.0"
openapi-types "^12.1.3"
p-retry "4"
uuid "^9.0.0"
uuid "^10.0.0"
yaml "^2.2.1"
zod "^3.22.4"
zod-to-json-schema "^3.22.3"
Expand Down Expand Up @@ -23864,7 +23857,19 @@ langchainhub@~0.0.8:
resolved "https://registry.yarnpkg.com/langchainhub/-/langchainhub-0.0.11.tgz#2ce22def9c84699dcbd4fd4b78270d34bd2a9ae9"
integrity sha512-WnKI4g9kU2bHQP136orXr2bcRdgz9iiTBpTN0jWt9IlScUKnJBoD0aa2HOzHURQKeQDnt2JwqVmQ6Depf5uDLQ==

langsmith@~0.1.30, langsmith@~0.1.7:
langsmith@^0.2.0:
version "0.2.5"
resolved "https://registry.yarnpkg.com/langsmith/-/langsmith-0.2.5.tgz#0e58900bcdfb47409222dd6c8c122c50f5015337"
integrity sha512-dA+l7ZEh1Q9Q9FcE39PUSSEMfsFo73R2V81fRo5KSlGNcypOEhoQvv6lbjyZP7MHmt3/9pPcfpuRd5Y4RbFYqQ==
dependencies:
"@types/uuid" "^10.0.0"
commander "^10.0.1"
p-queue "^6.6.2"
p-retry "4"
semver "^7.6.3"
uuid "^10.0.0"

langsmith@~0.1.30:
version "0.1.37"
resolved "https://registry.yarnpkg.com/langsmith/-/langsmith-0.1.37.tgz#7e7bf6fce3eab2a9e95221d5879820ec29d0aa60"
integrity sha512-8JgWykdJywdKWs+WeefOEf4Gkz3YdNkvG5u5JPbgXuodTUwuHPwjmblsldt1OGKkPp7iCWfdtCdnc9z9MYC/Dw==
Expand Down Expand Up @@ -27643,7 +27648,7 @@ open@^8.0.9, open@^8.4.0:
is-docker "^2.1.1"
is-wsl "^2.2.0"

openai@^4.41.1, openai@^4.49.1:
openai@^4.49.1:
version "4.52.7"
resolved "https://registry.yarnpkg.com/openai/-/openai-4.52.7.tgz#e32b000142287a9e8eda8512ba28df33d11ec1f1"
integrity sha512-dgxA6UZHary6NXUHEDj5TWt8ogv0+ibH+b4pT5RrWMjiRZVylNwLcw/2ubDrX5n0oUmHX/ZgudMJeemxzOvz7A==
Expand All @@ -27657,6 +27662,19 @@ openai@^4.41.1, openai@^4.49.1:
node-fetch "^2.6.7"
web-streams-polyfill "^3.2.1"

openai@^4.71.0:
version "4.71.1"
resolved "https://registry.yarnpkg.com/openai/-/openai-4.71.1.tgz#f26bf5db00d75703676d80bf0ae7cb7674e41eac"
integrity sha512-C6JNMaQ1eijM0lrjiRUL3MgThVP5RdwNAghpbJFdW0t11LzmyqON8Eh8MuUuEZ+CeD6bgYl2Fkn2BoptVxv9Ug==
dependencies:
"@types/node" "^18.11.18"
"@types/node-fetch" "^2.6.4"
abort-controller "^3.0.0"
agentkeepalive "^4.2.1"
form-data-encoder "1.7.2"
formdata-node "^4.3.2"
node-fetch "^2.6.7"

openapi-types@^12.1.3:
version "12.1.3"
resolved "https://registry.yarnpkg.com/openapi-types/-/openapi-types-12.1.3.tgz#471995eb26c4b97b7bd356aacf7b91b73e777dd3"
Expand Down Expand Up @@ -31466,6 +31484,11 @@ semver@^7.6.0:
resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.2.tgz#1e3b34759f896e8f14d6134732ce798aeb0c6e13"
integrity sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==

semver@^7.6.3:
version "7.6.3"
resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143"
integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==

semver@~7.0.0:
version "7.0.0"
resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e"
Expand Down

0 comments on commit da3eddf

Please sign in to comment.