diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index 8ce734a..b09cb06 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -7,31 +7,7 @@ on: permissions: contents: write -env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_TOKEN }} - COHERE_API_TOKEN: ${{ secrets.COHERE_API_TOKEN }} - ANTHROPIC_API_TOKEN: ${{ secrets.ANTHROPIC_API_TOKEN }} - DOKU_URL: ${{ secrets.DOKU_URL }} - DOKU_TOKEN: ${{ secrets.DOKU_TOKEN }} - jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v3 - with: - node-version: 20 - cache: 'npm' - - - run: npm ci - - - name: NPM Test - run: npm test - publish: needs: build runs-on: ubuntu-latest diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4729927..a9b07f6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,14 +7,14 @@ on: branches: [ "main" ] workflow_dispatch: schedule: - - cron: '0 9 * * *' + - cron: '0 0 * * 0' env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_TOKEN }} COHERE_API_TOKEN: ${{ secrets.COHERE_API_TOKEN }} ANTHROPIC_API_TOKEN: ${{ secrets.ANTHROPIC_API_TOKEN }} - DOKU_URL: ${{ secrets.DOKU_URL }} - DOKU_TOKEN: ${{ secrets.DOKU_TOKEN }} + MISTRAL_API_TOKEN: ${{ secrets.MISTRAL_API_TOKEN }} + DOKU_URL: http://127.0.0.1:9044 jobs: build: @@ -27,6 +27,24 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0 + + - name: Setup Doku Stack + run: docker-compose up -d + + - name: Sleep for 30 seconds + run: sleep 30 + + - name: Make API Request and Set DOKU_TOKEN + run: | + RESPONSE=$(curl -X POST $DOKU_URL/api/keys \ + -H 'Authorization: ""' \ + -H 'Content-Type: application/json' \ + -d '{"Name": "GITHUBACTION"}') + MESSAGE=$(echo $RESPONSE | jq -r '.message') + echo "DOKU_TOKEN=${MESSAGE}" >> $GITHUB_ENV + - name: Setup Node.js ${{ matrix.node-version }} uses: actions/setup-node@v3 with: diff --git a/README.md b/README.md index 1a1a033..c5e104c 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ - ✅ OpenAI - ✅ Anthropic - ✅ Cohere +- ✅ Mistral Deployed as the backbone for all your LLM monitoring needs, `dokumetry` channels crucial usage data directly to Doku, streamlining the tracking process. Unlock efficient and effective observability for your LLM applications with DokuMetry. diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..ebc8adf --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,52 @@ +version: '3.8' + +services: + clickhouse: + image: clickhouse/clickhouse-server:24.1.5 + container_name: clickhouse + environment: + CLICKHOUSE_PASSWORD: ${DOKU_DB_PASSWORD:-DOKU} + CLICKHOUSE_USER: ${DOKU_DB_USER:-default} + volumes: + - clickhouse-data:/var/lib/clickhouse + ports: + - "9000:9000" + - "8123:8123" + restart: always + + doku-ingester: + image: ghcr.io/dokulabs/doku-ingester:latest + container_name: doku-ingester + environment: + DOKU_DB_HOST: clickhouse + DOKU_DB_PORT: 9000 + DOKU_DB_NAME: ${DOKU_DB_NAME:-default} + DOKU_DB_USER: ${DOKU_DB_USER:-default} + DOKU_DB_PASSWORD: ${DOKU_DB_PASSWORD:-DOKU} + ports: + - "9044:9044" + depends_on: + - clickhouse + restart: always + + doku-client: + image: ghcr.io/dokulabs/doku-client:latest + container_name: doku-client + environment: + INIT_DB_HOST: clickhouse + INIT_DB_PORT: 8123 + INIT_DB_DATABASE: ${DOKU_DB_NAME:-default} + INIT_DB_USERNAME: ${DOKU_DB_USER:-default} + INIT_DB_PASSWORD: ${DOKU_DB_PASSWORD:-DOKU} + SQLITE_DATABASE_URL: file:/app/client/data/data.db + ports: + - "3000:3000" + depends_on: + - clickhouse + volumes: + - doku-client-data:/app/client/data + restart: always + +volumes: + clickhouse-data: + doku-client-data: \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index dd8f0b9..6451489 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,18 +1,20 @@ { "name": "dokumetry", - "version": "0.1.0", + "version": "0.1.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "dokumetry", - "version": "0.1.0", + "version": "0.1.1", "license": "Apache-2.0", "dependencies": { "stream": "^0.0.2" }, "devDependencies": { "@anthropic-ai/sdk": "^0.17.1", + "@azure/openai": "^1.0.0-beta.11", + "@mistralai/mistralai": "^0.1.3", "chai": "^5.0.3", "cohere-ai": "^7.7.3", "eslint": "^8.56.0", @@ -47,6 +49,135 @@ "web-streams-polyfill": "^3.2.1" } }, + "node_modules/@azure-rest/core-client": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@azure-rest/core-client/-/core-client-1.3.0.tgz", + "integrity": "sha512-OmAB+qbWZJk4p9+aqF3zM3J3J371RTdz1gRvz4uxl/+MGLKfKBMzZqVkAUIY8h1qzux4ypozCiRPJ3wdWyPDUg==", + "dev": true, + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-auth": "^1.3.0", + "@azure/core-rest-pipeline": "^1.5.0", + "@azure/core-tracing": "^1.0.1", + "@azure/core-util": "^1.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/abort-controller": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.0.tgz", + "integrity": "sha512-SYtcG13aiV7znycu6plCClWUzD9BBtfnsbIxT89nkkRvQRB4n0kuZyJJvJ7hqdKOn7x7YoGKZ9lVStLJpLnOFw==", + "dev": true, + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-auth": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.7.0.tgz", + "integrity": "sha512-OuDVn9z2LjyYbpu6e7crEwSipa62jX7/ObV/pmXQfnOG8cHwm363jYtg3FSX3GB1V7jsIKri1zgq7mfXkFk/qw==", + "dev": true, + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-util": "^1.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-rest-pipeline": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.15.0.tgz", + "integrity": "sha512-6kBQwE75ZVlOjBbp0/PX0fgNLHxoMDxHe3aIPV/RLVwrIDidxTbsHtkSbPNTkheMset3v9s1Z08XuMNpWRK/7w==", + "dev": true, + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-auth": "^1.4.0", + "@azure/core-tracing": "^1.0.1", + "@azure/core-util": "^1.3.0", + "@azure/logger": "^1.0.0", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-sse": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@azure/core-sse/-/core-sse-2.1.0.tgz", + "integrity": "sha512-wH5FEaaAYxfCtkbJQ07BjLhIpjyuVutaDbMvtsFmOfkQHZGNf+3VMMaFAkQG2S2qGaIahkLxOzp/zJ3KEqATYw==", + "dev": true, + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-tracing": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.1.0.tgz", + "integrity": "sha512-MVeJvGHB4jmF7PeHhyr72vYJsBJ3ff1piHikMgRaabPAC4P3rxhf9fm42I+DixLysBunskJWhsDQD2A+O+plkQ==", + "dev": true, + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-util": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.8.0.tgz", + "integrity": "sha512-w8NrGnrlGDF7fj36PBnJhGXDK2Y3kpTOgL7Ksb5snEHXq/3EAbKYOp1yqme0yWCUlSDq5rjqvxSBAJmsqYac3w==", + "dev": true, + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/logger": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.1.0.tgz", + "integrity": "sha512-BnfkfzVEsrgbVCtqq0RYRMePSH2lL/cgUUR5sYRF4yNN10zJZq/cODz0r89k3ykY83MqeM3twR292a3YBNgC3w==", + "dev": true, + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/openai": { + "version": "1.0.0-beta.11", + "resolved": "https://registry.npmjs.org/@azure/openai/-/openai-1.0.0-beta.11.tgz", + "integrity": "sha512-OXS27xkG1abiGf5VZUKnkJKr1VCo8+6EUrTGW5aSVjc5COqX8jAUqVAOZsQVCHBdtWYSBULlZkc0ncKMTRQAiQ==", + "dev": true, + "dependencies": { + "@azure-rest/core-client": "^1.1.7", + "@azure/core-auth": "^1.4.0", + "@azure/core-rest-pipeline": "^1.13.0", + "@azure/core-sse": "^2.0.0", + "@azure/core-util": "^1.4.0", + "@azure/logger": "^1.0.3", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", @@ -136,6 +267,15 @@ "integrity": "sha512-6EwiSjwWYP7pTckG6I5eyFANjPhmPjUX9JRLUSfNPC7FX7zK9gyZAfUEaECL6ALTpGX5AjnBq3C9XmVWPitNpw==", "dev": true }, + "node_modules/@mistralai/mistralai": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@mistralai/mistralai/-/mistralai-0.1.3.tgz", + "integrity": "sha512-WUHxC2xdeqX9PTXJEqdiNY54vT2ir72WSJrZTTBKRnkfhX6zIfCYA24faRlWjUB5WTpn+wfdGsTMl3ArijlXFA==", + "dev": true, + "dependencies": { + "node-fetch": "^2.6.7" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -229,6 +369,18 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/agent-base": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz", + "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==", + "dev": true, + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/agentkeepalive": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz", @@ -1245,6 +1397,32 @@ "he": "bin/he" } }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "dev": true, + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz", + "integrity": "sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==", + "dev": true, + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/humanize-ms": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", @@ -2154,6 +2332,12 @@ "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", "dev": true }, + "node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", + "dev": true + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/package.json b/package.json index dc5d485..5d46d5b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dokumetry", - "version": "0.1.0", + "version": "0.1.1", "description": "An NPM Package for tracking OpenAI API calls and sending usage metrics to Doku", "main": "src/index.js", "scripts": { @@ -27,7 +27,9 @@ "stream": "^0.0.2" }, "devDependencies": { + "@azure/openai": "^1.0.0-beta.11", "@anthropic-ai/sdk": "^0.17.1", + "@mistralai/mistralai": "^0.1.3", "chai": "^5.0.3", "cohere-ai": "^7.7.3", "eslint": "^8.56.0", diff --git a/src/azure_openai.js b/src/azure_openai.js new file mode 100644 index 0000000..78776ae --- /dev/null +++ b/src/azure_openai.js @@ -0,0 +1,342 @@ +import {sendData} from './helpers.js'; +import { Readable } from 'stream'; + +/** + * Initializes Azure OpenAI functionality with performance tracking and data logging. + * + * @param {Object} llm - The Azure OpenAI function object. + * @param {string} dokuUrl - The URL for logging data. + * @param {string} apiKey - The authentication apiKey. + * @param {string} environment - The environment. + * @param {string} applicationName - The application name. + * @param {boolean} skipResp - To skip waiting for API resopnse. + * @return {void} + * + * @jsondoc + * { + * "description": "Performance tracking for Azure OpenAI APIs", + * "params": [ + * {"name": "llm", "type": "Object", "description": "Azure OpenAI function."}, + * {"name": "dokuUrl", "type": "string", "description": "The URL"}, + * {"name": "apiKey", "type": "string", "description": "The auth apiKey."}, + * {"name": "environment", "type": "string", "description": "The environment."}, + * {"name": "applicationName", "type": "string", "description": "The application name."}, + * {"name": "skipResp", "type": "boolean", "description": "To skip waiting for API resopnse."} + * ], + * "returns": {"type": "void"}, + * "example": { + * "description": "Example usage of init function.", + * "code": "init(azureOenaiFunc, 'https://example.com/log', 'authToken');" + * } + * } + */ +export default function initAzureOpenAI({ llm, dokuUrl, apiKey, environment, applicationName, skipResp }) { + // Save original method + const originalChatCreate = llm.chat.completions.create; + const originalCompletionsCreate = llm.completions.create; + const originalEmbeddingsCreate = llm.embeddings.create; + const originalImagesCreate = llm.images.generate; + + // Define wrapped method + llm.chat.completions.create = async function(params) { + const start = performance.now(); + let streaming = params.stream || false; + if (streaming) { + // Call original method + const originalResponseStream = await originalChatCreate.call(this, params); + + // Create a pass-through stream + const passThroughStream = new Readable({ + read() {}, + objectMode: true // Set to true because the chunks are objects + }); + + let dataResponse = ''; + let chatModel = ''; + + // Immediately-invoked async function to handle streaming + (async () => { + for await (const chunk of originalResponseStream) { + var content = chunk.choices[0]?.delta?.content; + if (content) { + dataResponse += content; + passThroughStream.push(chunk); // Push chunk to the pass-through stream + } + var responseId = chunk.id; + chatModel = chunk.model; + } + passThroughStream.push(null); // Signal end of the pass-through stream + + // Process response data after stream has ended + const end = performance.now(); + const duration = (end - start) / 1000; + + let formattedMessages = []; + for (let message of params.messages) { + let role = message.role; + let content = message.content; + + if (Array.isArray(content)) { + let contentStr = content.map(item => { + if (item.type) { + return `${item.type}: ${item.text || item.image_url}`; + } else { + return `text: ${item.text}`; + } + }).join(", "); + formattedMessages.push(`${role}: ${contentStr}`); + } else { + formattedMessages.push(`${role}: ${content}`); + } + } + let prompt = formattedMessages.join("\n"); + + // Prepare the data object for Doku + const data = { + llmReqId: responseId, + environment: environment, + applicationName: applicationName, + sourceLanguage: 'Javascript', + endpoint: 'azure.chat.completions', + skipResp: skipResp, + requestDuration: duration, + model: "azure_" + chatModel, + prompt: prompt, + response: dataResponse + }; + + await sendData(data, dokuUrl, apiKey); + })(); + + // Return the pass-through stream to the original caller + return passThroughStream; + } + else { + // Call original method + const response = await originalChatCreate.call(this, params); + const end = performance.now(); + const duration = (end - start) / 1000; + + let formattedMessages = []; + for (let message of params.messages) { + let role = message.role; + let content = message.content; + + if (Array.isArray(content)) { + let contentStr = content.map(item => { + if (item.type) { + return `${item.type}: ${item.text || item.image_url}`; + } else { + return `text: ${item.text}`; + } + }).join(", "); + formattedMessages.push(`${role}: ${contentStr}`); + } else { + formattedMessages.push(`${role}: ${content}`); + } + } + let prompt = formattedMessages.join("\n"); + const data = { + llmReqId: response.id, + environment: environment, + applicationName: applicationName, + sourceLanguage: 'Javascript', + endpoint: 'azure.chat.completions', + skipResp: skipResp, + requestDuration: duration, + model: "azure_" + response.model, + prompt: prompt, + }; + + if (!params.hasOwnProperty('tools')) { + data.completionTokens = response.usage.completion_tokens; + data.promptTokens = response.usage.prompt_tokens; + data.totalTokens = response.usage.total_tokens; + data.finishReason = response.choices[0].finish_reason; + + if (!params.hasOwnProperty('n') || params.n === 1) { + data.response = response.choices[0].message.content; + } else { + let i = 0; + while (i < params.n && i < response.choices.length) { + data.response = response.choices[i].message.content; + i++; + await sendData(data, dokuUrl, apiKey); + } + return response; + } + } else if (params.hasOwnProperty('tools')) { + data.response = "Function called with tools"; + data.completionTokens = response.usage.completion_tokens; + data.promptTokens = response.usage.prompt_tokens; + data.totalTokens = response.usage.total_tokens; + } + + await sendData(data, dokuUrl, apiKey); + + return response; + } + }; + + llm.completions.create = async function(params) { + const start = performance.now(); + let streaming = params.stream || false; + if (streaming) { + // Call original method + const originalResponseStream = await originalCompletionsCreate.call(this, params); + + // Create a pass-through stream + const passThroughStream = new Readable({ + read() {}, + objectMode: true // Set to true because the chunks are objects + }); + + let dataResponse = ''; + let chatModel = ''; + + // Immediately-invoked async function to handle streaming + (async () => { + for await (const chunk of originalResponseStream) { + if (chunk.choices.length > 0) { + dataResponse += chunk.choices[0].text; + passThroughStream.push(chunk); // Push chunk to the pass-through stream + } + var responseId = chunk.id; + chatModel = chunk.model; + } + passThroughStream.push(null); // Signal end of the pass-through stream + + // Process response data after stream has ended + const end = performance.now(); + const duration = (end - start) / 1000; + // Prepare the data object for Doku + const data = { + llmReqId: responseId, + environment: environment, + applicationName: applicationName, + sourceLanguage: 'Javascript', + endpoint: 'azure.completions', + skipResp: skipResp, + requestDuration: duration, + model: "azure_" + chatModel, + prompt: params.prompt, + response: dataResponse + }; + + await sendData(data, dokuUrl, apiKey); + })(); + + // Return the pass-through stream to the original caller + return passThroughStream; + } + else { + const response = await originalCompletionsCreate.call(this, params); + const end = performance.now(); + const duration = (end - start) / 1000; + + const data = { + llmReqId: response.id, + environment: environment, + applicationName: applicationName, + sourceLanguage: 'Javascript', + endpoint: 'azure.completions', + skipResp: skipResp, + requestDuration: duration, + model: "azure_" + response.model, + prompt: params.prompt, + }; + + if (!params.hasOwnProperty('tools')) { + data.completionTokens = response.usage.completion_tokens; + data.promptTokens = response.usage.prompt_tokens; + data.totalTokens = response.usage.total_tokens; + data.finishReason = response.choices[0].finish_reason; + + if (!params.hasOwnProperty('n') || params.n === 1) { + data.response = response.choices[0].text; + } else { + let i = 0; + while (i < params.n && i < response.choices.length) { + data.response = response.choices[i].text; + i++; + + await sendData(data, dokuUrl, apiKey); + } + return response; + } + } else if (params.hasOwnProperty('tools')) { + data.response = "Function called with tools"; + data.completionTokens = response.usage.completion_tokens; + data.promptTokens = response.usage.prompt_tokens; + data.totalTokens = response.usage.total_tokens; + } + + await sendData(data, dokuUrl, apiKey); + + return response; + } + }; + + llm.embeddings.create = async function(params) { + const start = performance.now(); + const response = await originalEmbeddingsCreate.call(this, params); + const end = performance.now(); + const duration = (end - start) / 1000; + + const data = { + environment: environment, + applicationName: applicationName, + sourceLanguage: 'Javascript', + endpoint: 'azure.embeddings', + skipResp: skipResp, + requestDuration: duration, + model: "azure_" + response.model, + prompt: params.input, + promptTokens: response.usage.prompt_tokens, + totalTokens: response.usage.total_tokens, + }; + + await sendData(data, dokuUrl, apiKey); + + return response; + }; + + llm.images.generate = async function(params) { + const start = performance.now(); + const response = await originalImagesCreate.call(this, params); + const end = performance.now(); + const duration = (end - start) / 1000; + const size = params.size || '1024x1024'; + const model = 'azure_dall-e-3'; + let imageFormat = 'url'; + + if (params.response_format && params.response_format === 'b64_json') { + imageFormat = 'b64_json'; + } + + const quality = params.quality ?? 'standard'; + var responseId = response.created; + for (const item of response.data) { + const data = { + llmReqId: responseId, + environment: environment, + applicationName: applicationName, + sourceLanguage: 'Javascript', + endpoint: 'azure.images.create', + skipResp: skipResp, + requestDuration: duration, + model: model, + prompt: params.prompt, + imageSize: size, + imageQuality: quality, + revisedPrompt: item.revised_prompt || null, + image: item[imageFormat], + }; + + await sendData(data, dokuUrl, apiKey); + } + + return response; + }; + +} diff --git a/src/cohere.js b/src/cohere.js index 2b3c990..014dbe4 100644 --- a/src/cohere.js +++ b/src/cohere.js @@ -66,8 +66,8 @@ export default function initCohere({ llm, dokuUrl, apiKey, environment, applicat if (!params.hasOwnProperty('stream') || params.stream !== true) { data.finishReason = generation.finish_reason; } - console.log(data); - //await sendData(data, dokuUrl, apiKey); + + await sendData(data, dokuUrl, apiKey); } return response; @@ -92,6 +92,7 @@ export default function initCohere({ llm, dokuUrl, apiKey, environment, applicat model: model, prompt: prompt, promptTokens: response.meta["billedUnits"]["inputTokens"], + totalTokens: response.meta["billedUnits"]["inputTokens"], }; await sendData(data, dokuUrl, apiKey); diff --git a/src/index.js b/src/index.js index e8caeba..f26ebe4 100644 --- a/src/index.js +++ b/src/index.js @@ -1,6 +1,8 @@ import initOpenAI from './openai.js'; import initCohere from './cohere.js'; import initAnthropic from './anthropic.js'; +import initMistral from './mistral.js'; +import initAzureOpenAI from './azure_openai.js'; /** * Represents the configuration for Doku. @@ -52,12 +54,16 @@ function init({ llm, dokuUrl, apiKey, environment="default", applicationName="de DokuConfig.applicationName = applicationName; DokuConfig.skipResp = skipResp; - if (llm.fineTuning && typeof llm.completions.create === 'function') { + if (llm.fineTuning && typeof llm.completions.create === 'function' && !(llm.baseURL.includes('azure.com'))) { initOpenAI({ llm, dokuUrl, apiKey, environment, applicationName, skipResp }); + } else if (llm.fineTuning && typeof llm.completions.create === 'function' && llm.baseURL.includes('azure.com')) { + initAzureOpenAI({ llm, dokuUrl, apiKey, environment, applicationName, skipResp }); } else if (llm.generate && typeof llm.rerank === 'function') { initCohere({ llm, dokuUrl, apiKey, environment, applicationName, skipResp }); } else if (llm.messages && typeof llm.messages.create === 'function') { initAnthropic({ llm, dokuUrl, apiKey, environment, applicationName, skipResp }); + } else if (llm.listModels && typeof llm.chatStream === 'function') { + initMistral({ llm, dokuUrl, apiKey, environment, applicationName, skipResp }); } } diff --git a/src/mistral.js b/src/mistral.js new file mode 100644 index 0000000..aed7be0 --- /dev/null +++ b/src/mistral.js @@ -0,0 +1,171 @@ +import {sendData} from './helpers.js'; + +/** + * Initializes Mistral functionality with performance tracking. + * + * @param {Object} llm - The Mistral function object. + * @param {string} dokuUrl - The URL for logging data. + * @param {string} apiKey - The authentication apiKey. + * @param {string} environment - The environment. + * @param {string} applicationName - The application name. + * @param {boolean} skipResp - To skip waiting for API resopnse. + * @return {void} + * + * @jsondoc + * { + * "description": "Initializes Mistral function and performance tracking", + * "params": [ + * {"name": "dokuUrl", "type": "string", "description": "Doku URL"}, + * {"name": "apiKey", "type": "string", "description": "Auth apiKey"}, + * {"name": "llm", "type": "Object", "description": "The Mistral object"}, + * {"name": "environment", "type": "string", "description": "Environment"}, + * {"name": "applicationName", "type": "string", "description": "Application Name"}, + * {"name": "skipResp", "type": "boolean", "description": "To skip waiting for API resopnse."} + * ], + * "returns": {"type": "void"}, + * "example": { + * "description": "Example usage of init function.", + * "code": "init('https://example.com/log', 'authToken', mistralFunc);" + * } + * } + */ +export default function initMistral({ llm, dokuUrl, apiKey, environment, applicationName, skipResp }) { + const origianlMistralChat = llm.chat; + const origianlMistralChatStream = llm.chatStream; + const originalMistralEmbedding = llm.embeddings; + + // Define wrapped method + llm.chat = async function(params) { + const start = performance.now(); + const response = await origianlMistralChat.call(this, params); + const end = performance.now(); + const duration = (end - start) / 1000; + let formattedMessages = []; + for (let message of params.messages) { + let role = message.role; + let content = message.content; + + if (Array.isArray(content)) { + let contentStr = content.map(item => { + if (item.type) { + return `${item.type}: ${item.text || item.image_url}`; + } else { + return `text: ${item.text}`; + } + }).join(", "); + formattedMessages.push(`${role}: ${contentStr}`); + } else { + formattedMessages.push(`${role}: ${content}`); + } + } + let prompt = formattedMessages.join("\n"); + + const data = { + llmReqId: response.id, + environment: environment, + applicationName: applicationName, + sourceLanguage: 'Javascript', + endpoint: 'mistral.chat', + skipResp: skipResp, + completionTokens: response.usage.prompt_tokens, + promptTokens: response.usage.completion_tokens, + totalTokens: response.usage.total_tokens, + requestDuration: duration, + model: params.model, + prompt: String(prompt), + finishReason: response.choices[0].finish_reason, + response: String(response.choices[0].message.content), + }; + + await sendData(data, dokuUrl, apiKey); + + return response; + }; + + llm.chatStream = async function* (params) { + const start = performance.now(); + const response = await origianlMistralChatStream.call(this, params); + + const model = params.model || 'mistral-large-latest'; + let formattedMessages = []; + for (let message of params.messages) { + let role = message.role; + let content = message.content; + + if (Array.isArray(content)) { + let contentStr = content.map(item => { + if (item.type) { + return `${item.type}: ${item.text || item.image_url}`; + } else { + return `text: ${item.text}`; + } + }).join(", "); + formattedMessages.push(`${role}: ${contentStr}`); + } else { + formattedMessages.push(`${role}: ${content}`); + } + } + let prompt = formattedMessages.join("\n"); + + const data = { + environment: environment, + applicationName: applicationName, + sourceLanguage: 'Javascript', + endpoint: 'mistral.chat', + skipResp: skipResp, + model: model, + prompt: prompt, + }; + + data.response = "" + for await (const message of response) { + data.llmReqId = message.id; + data.response += message.choices[0].delta.content + if (message.choices[0].finish_reason != null) { + data.promptTokens = message.usage.prompt_tokens; + data.completionTokens = message.usage.completion_tokens; + data.totalTokens = message.usage.total_tokens; + data.finishReason = message.choices[0].finish_reason; + } + // Pass the message along so it's not consumed + yield message; // this allows the message to flow back to the original caller + } + + const end = performance.now(); + data.requestDuration = (end - start) / 1000; + + await sendData(data, dokuUrl, apiKey); + + return response; + }; + + llm.embeddings = async function(params) { + const start = performance.now(); + const response = await originalMistralEmbedding.call(this, params); + const end = performance.now(); + const duration = (end - start) / 1000; + + const model = params.model || 'mistral-embed'; + const prompt = params.input.toString(); + + const data = { + llmReqId: response.id, + environment: environment, + applicationName: applicationName, + sourceLanguage: 'Javascript', + endpoint: 'mistral.embeddings', + skipResp: skipResp, + requestDuration: duration, + model: model, + prompt: prompt, + promptTokens: response.usage.prompt_tokens, + completionTokens: response.usage.completion_tokens, + totalTokens: response.usage.total_tokens, + }; + + await sendData(data, dokuUrl, apiKey); + + return response; + }; + +} diff --git a/src/openai.js b/src/openai.js index d56ae8d..534d579 100644 --- a/src/openai.js +++ b/src/openai.js @@ -184,7 +184,7 @@ export default function initOpenAI({ llm, dokuUrl, apiKey, environment, applicat let streaming = params.stream || false; if (streaming) { // Call original method - const originalResponseStream = await originalChatCreate.call(this, params); + const originalResponseStream = await originalCompletionsCreate.call(this, params); // Create a pass-through stream const passThroughStream = new Readable({ diff --git a/tests/anthropic.test.mjs b/tests/anthropic.test.mjs deleted file mode 100644 index df83287..0000000 --- a/tests/anthropic.test.mjs +++ /dev/null @@ -1,20 +0,0 @@ -import Anthropic from '@anthropic-ai/sdk'; -import {expect} from 'chai'; -import DokuMetry from '../src/index.js'; - -describe('Anthropic Test', () => { - const anthropic = new Anthropic({ - apiKey: process.env.ANTHROPIC_API_TOKEN, - }); - - it('should return a response with type as "message"', async () => { - DokuMetry.init({llm: anthropic, dokuUrl: process.env.DOKU_URL, apiKey: process.env.DOKU_TOKEN, environment: "dokumetry-testing", applicationName: "dokumetry-node-test", skipResp: false}); - const message = await anthropic.messages.create({ - model: "claude-3-opus-20240229", - max_tokens: 1024, - messages: [{ role: "user", content: "Hello, Doku!" }], - }); - - expect(message.type).to.equal('message'); - }).timeout(10000); -}); \ No newline at end of file diff --git a/tests/anthropic.test.mjs.hold b/tests/anthropic.test.mjs.hold new file mode 100644 index 0000000..f09a3c7 --- /dev/null +++ b/tests/anthropic.test.mjs.hold @@ -0,0 +1,39 @@ +import Anthropic from '@anthropic-ai/sdk'; +import {expect} from 'chai'; +import DokuMetry from '../src/index.js'; + +describe('Anthropic Test', () => { + const anthropic = new Anthropic({ + apiKey: process.env.ANTHROPIC_API_TOKEN, + }); + + // Non-streaming messages + it('should return a response with type as "message"', async () => { + DokuMetry.init({llm: anthropic, dokuUrl: process.env.DOKU_URL, apiKey: process.env.DOKU_TOKEN, environment: "dokumetry-testing", applicationName: "dokumetry-node-test", skipResp: false}); + const message = await anthropic.messages.create({ + model: "claude-3-opus-20240229", + max_tokens: 100, + messages: [{ role: "user", content: "How to monitor LLM Applications in one sentence?" }], + }); + + expect(message.type).to.equal('message'); + }).timeout(30000); + + it('should return a response with type as "message"', async () => { + DokuMetry.init({llm: anthropic, dokuUrl: process.env.DOKU_URL, apiKey: process.env.DOKU_TOKEN, environment: "dokumetry-testing", applicationName: "dokumetry-node-test", skipResp: false}); + var stream = await anthropic.messages.create({ + max_tokens: 100, + messages: [{ role: 'user', content: 'How to monitor LLM Applications in one sentence?' }], + model: 'claude-3-opus-20240229', + stream: true, + }); + for await (const messageStreamEvent of stream) { + if (messageStreamEvent.type === 'message_start') { + expect(messageStreamEvent.type).to.equal('message_start'); + } + if (messageStreamEvent.type === 'content_block_delta') { + expect(messageStreamEvent.type).to.equal('content_block_delta'); + } + } + }).timeout(30000); +}); \ No newline at end of file diff --git a/tests/azure-openai.test.mjs.hold b/tests/azure-openai.test.mjs.hold new file mode 100644 index 0000000..5c98944 --- /dev/null +++ b/tests/azure-openai.test.mjs.hold @@ -0,0 +1,92 @@ +import OpenAI from 'openai'; +import {expect} from 'chai'; +import DokuMetry from '../src/index.js'; +import fs from "fs"; + +describe('OpenAI Test', () => { + let openai; + + before(async () => { + openai = new OpenAI({ + apiKey: process.env.AZURE_OPENAI_API_KEY, + }); + }); + + it('should return a response with object as "chat.completion"', async () => { + await DokuMetry.init({llm: openai, dokuUrl: process.env.DOKU_URL, apiKey: process.env.DOKU_TOKEN, environment: "dokumetry-testing", applicationName: "dokumetry-node-test", skipResp: false}); + const chatCompletion = await openai.chat.completions.create({ + messages: [{role: 'user', content: 'What is LLM Monitoring?'}], + model: 'gpt-3.5-turbo', + }); + + expect(chatCompletion.object).to.equal('chat.completion'); + }).timeout(30000);; + + it('should return a response with object as "text_completion"', async () => { + await DokuMetry.init({llm: openai, dokuUrl: process.env.DOKU_URL, apiKey: process.env.DOKU_TOKEN, environment: "dokumetry-testing", applicationName: "dokumetry-node-test", skipResp: false}); + const completion = await openai.completions.create({ + model: 'gpt-3.5-turbo-instruct', + prompt: 'What is LLM Observability?', + max_tokens: 7, + }); + + expect(completion.object).to.equal('text_completion'); + }).timeout(30000);; + + it('should return a response with object as "embedding"', async () => { + await DokuMetry.init({llm: openai, dokuUrl: process.env.DOKU_URL, apiKey: process.env.DOKU_TOKEN, environment: "dokumetry-testing", applicationName: "dokumetry-node-test", skipResp: false}); + const embeddings = await openai.embeddings.create({ + model: 'text-embedding-ada-002', + input: 'The quick brown fox jumped over the lazy dog', + encoding_format: 'float', + }); + + expect(embeddings.data[0].object).to.equal('embedding'); + }).timeout(30000);; + + it('should return a response with object as "fine_tuning.job"', async () => { + await DokuMetry.init({llm: openai, dokuUrl: process.env.DOKU_URL, apiKey: process.env.DOKU_TOKEN, environment: "dokumetry-testing", applicationName: "dokumetry-node-test", skipResp: false}); + try { + const fineTuningJob = await openai.fineTuning.jobs.create({ + training_file: 'file-m36cc45komO83VJKAY1qVgeP', + model: 'gpt-3.5-turbo', + }); + + expect(fineTuningJob.object).to.equal('fine_tuning.job'); + } catch (error) { + // Check if it's a rate limit error + if (error.code == "daily_rate_limit_exceeded") { + console.error(`Daily Rate limit Reached`); + } + } + }).timeout(10000); + + it('should return a response with "created" field', async () => { + await DokuMetry.init({llm: openai, dokuUrl: process.env.DOKU_URL, apiKey: process.env.DOKU_TOKEN, environment: "dokumetry-testing", applicationName: "dokumetry-node-test", skipResp: false}); + const imageGeneration = await openai.images.generate({ + model: 'dall-e-2', + prompt: 'Generate an image of a cat.', + }); + + expect(imageGeneration.created).to.exist; + }).timeout(30000); + + it('should return a response with "created" field', async () => { + await DokuMetry.init({llm: openai, dokuUrl: process.env.DOKU_URL, apiKey: process.env.DOKU_TOKEN, environment: "dokumetry-testing", applicationName: "dokumetry-node-test", skipResp: false}); + const imageVariation = await openai.images.createVariation({ + image: fs.createReadStream('tests/test-image-for-openai.png'), + }); + + expect(imageVariation.created).to.exist; + }).timeout(30000); + + it('should return a response with url as "https://api.openai.com/v1/audio/speech"', async () => { + DokuMetry.init({llm: openai, dokuUrl: process.env.DOKU_URL, apiKey: process.env.DOKU_TOKEN, environment: "dokumetry-testing", applicationName: "dokumetry-node-test", skipResp: false}); + const audioSpeech = await openai.audio.speech.create({ + model: 'tts-1', + voice: 'alloy', + input: 'Today is a wonderful day to build something people love!', + }); + expect(audioSpeech.url).to.equal('https://api.openai.com/v1/audio/speech'); + }).timeout(30000); +}); diff --git a/tests/cohere.test.mjs b/tests/cohere.test.mjs index eceefb5..8e399a6 100644 --- a/tests/cohere.test.mjs +++ b/tests/cohere.test.mjs @@ -48,7 +48,7 @@ describe('Cohere Test', () => { DokuMetry.init({llm: cohere, dokuUrl: process.env.DOKU_URL, apiKey: process.env.DOKU_TOKEN, environment: "dokumetry-testing", applicationName: "dokumetry-node-test", skipResp: false}); try { const generate = await cohere.generate({ - prompt: 'Doku', + prompt: 'How to monitor LLM Applications?', maxTokens: 10, }); @@ -65,7 +65,7 @@ describe('Cohere Test', () => { DokuMetry.init({llm: cohere, dokuUrl: process.env.DOKU_URL, apiKey: process.env.DOKU_TOKEN, environment: "dokumetry-testing", applicationName: "dokumetry-node-test", skipResp: false}); try { const embeddings = await cohere.embed({ - texts: ['This is a test'], + texts: ['What is AI Observability?'], model: 'embed-multilingual-v2.0', }); expect(embeddings.id).to.exist; @@ -81,7 +81,7 @@ describe('Cohere Test', () => { DokuMetry.init({llm: cohere, dokuUrl: process.env.DOKU_URL, apiKey: process.env.DOKU_TOKEN, environment: "dokumetry-testing", applicationName: "dokumetry-node-test", skipResp: false}); try { const chatResponse = await cohere.chat({ - message: 'Say this is a test', + message: 'How to monitor AI Applications?', model: 'command', }); diff --git a/tests/mistral.test.mjs.hold b/tests/mistral.test.mjs.hold new file mode 100644 index 0000000..669da28 --- /dev/null +++ b/tests/mistral.test.mjs.hold @@ -0,0 +1,32 @@ +import MistralClient from '@mistralai/mistralai'; +import {expect} from 'chai'; +import DokuMetry from '../src/index.js'; + +describe('Mistral Test', () => { + const client = new MistralClient(process.env.MISTRAL_API_TOKEN); + + it('should return a response with object as "list"', async () => { + DokuMetry.init({llm: client, dokuUrl: process.env.DOKU_URL, apiKey: process.env.DOKU_TOKEN, environment: "dokumetry-testing", applicationName: "dokumetry-node-test", skipResp: false}); + const input = []; + for (let i = 0; i < 1; i++) { + input.push('What is the best French cheese?'); + } + + const message = await client.embeddings({ + model: 'mistral-embed', + input: input, + }); + + expect(message.object).to.equal('list'); + }).timeout(30000); + + it('should return a response with object as "chat.completion"', async () => { + DokuMetry.init({llm: client, dokuUrl: process.env.DOKU_URL, apiKey: process.env.DOKU_TOKEN, environment: "dokumetry-testing", applicationName: "dokumetry-node-test", skipResp: false}); + const message = await client.chat({ + model: 'mistral-large-latest', + messages: [{role: 'user', content: 'What is LLM Observability?'}], + }); + + expect(message.object).to.equal('chat.completion'); + }).timeout(30000); +}); \ No newline at end of file diff --git a/tests/openai.test.mjs b/tests/openai.test.mjs index 8e8623b..d5f2d40 100644 --- a/tests/openai.test.mjs +++ b/tests/openai.test.mjs @@ -15,23 +15,23 @@ describe('OpenAI Test', () => { it('should return a response with object as "chat.completion"', async () => { await DokuMetry.init({llm: openai, dokuUrl: process.env.DOKU_URL, apiKey: process.env.DOKU_TOKEN, environment: "dokumetry-testing", applicationName: "dokumetry-node-test", skipResp: false}); const chatCompletion = await openai.chat.completions.create({ - messages: [{role: 'user', content: 'Say this is a test'}], + messages: [{role: 'user', content: 'What is LLM Monitoring?'}], model: 'gpt-3.5-turbo', }); expect(chatCompletion.object).to.equal('chat.completion'); - }); + }).timeout(30000);; it('should return a response with object as "text_completion"', async () => { await DokuMetry.init({llm: openai, dokuUrl: process.env.DOKU_URL, apiKey: process.env.DOKU_TOKEN, environment: "dokumetry-testing", applicationName: "dokumetry-node-test", skipResp: false}); const completion = await openai.completions.create({ model: 'gpt-3.5-turbo-instruct', - prompt: 'Say this is a test.', + prompt: 'What is LLM Observability?', max_tokens: 7, }); expect(completion.object).to.equal('text_completion'); - }); + }).timeout(30000);; it('should return a response with object as "embedding"', async () => { await DokuMetry.init({llm: openai, dokuUrl: process.env.DOKU_URL, apiKey: process.env.DOKU_TOKEN, environment: "dokumetry-testing", applicationName: "dokumetry-node-test", skipResp: false}); @@ -42,7 +42,7 @@ describe('OpenAI Test', () => { }); expect(embeddings.data[0].object).to.equal('embedding'); - }); + }).timeout(30000);; it('should return a response with object as "fine_tuning.job"', async () => { await DokuMetry.init({llm: openai, dokuUrl: process.env.DOKU_URL, apiKey: process.env.DOKU_TOKEN, environment: "dokumetry-testing", applicationName: "dokumetry-node-test", skipResp: false});