plugins(cartesia): init with TTS (#217)

Co-authored-by: Théo Monnom <[email protected]>

.changeset/tiny-cameras-sort.md

@@ -0,0 +1,5 @@
+---
+"@livekit/agents-plugin-cartesia": minor
+---
+
+plugins(cartesia): init with TTS

.github/workflows/test.yml

@@ -44,13 +44,19 @@ jobs:
         run: pnpm test agents
       - name: Test all plugins
         if: steps.filter.outputs.agents-or-tests == 'true' || github.event_name != 'pull_request'
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          ELEVEN_API_KEY: ${{ secrets.ELEVEN_API_KEY }}
+          DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }}
+          CARTESIA_API_KEY: ${{ secrets.CARTESIA_API_KEY }}
         run: pnpm test plugins
       - name: Test specific plugins
         if: steps.check.outputs.agents-or-tests == 'false' && steps.check.outputs.plugins == 'true' && github.event_name == 'pull_request'
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           ELEVEN_API_KEY: ${{ secrets.ELEVEN_API_KEY }}
           DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }}
+          CARTESIA_API_KEY: ${{ secrets.CARTESIA_API_KEY }}
         run: |
           plugins=$(git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep '^plugins.*\.ts$' | cut -d/ -f2 | sort -u | tr '\n' ' ')
           read -ra plugins <<< "$plugins"

plugins/cartesia/README.md

@@ -0,0 +1,17 @@
+<!--
+SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+
+SPDX-License-Identifier: Apache-2.0
+-->
+# Cartesia plugin for LiveKit Agents
+
+The Agents Framework is designed for building realtime, programmable
+participants that run on servers. Use it to create conversational, multi-modal
+voice agents that can see, hear, and understand.
+
+This package contains the Cartesia plugin, which allows for voice synthesis.
+Refer to the [documentation](https://docs.livekit.io/agents/overview/) for
+information on how to use it, or browse the [API
+reference](https://docs.livekit.io/agents-js/modules/plugins_agents_plugin_cartesia.html).
+See the [repository](https://github.com/livekit/agents-js) for more information
+about the framework as a whole.

plugins/cartesia/api-extractor.json

@@ -0,0 +1,20 @@
+/**
+ * Config file for API Extractor.  For more info, please visit: https://api-extractor.com
+ */
+{
+  "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json",
+
+  /**
+   * Optionally specifies another JSON config file that this file extends from.  This provides a way for
+   * standard settings to be shared across multiple projects.
+   *
+   * If the path starts with "./" or "../", the path is resolved relative to the folder of the file that contains
+   * the "extends" field.  Otherwise, the first path segment is interpreted as an NPM package name, and will be
+   * resolved using NodeJS require().
+   *
+   * SUPPORTED TOKENS: none
+   * DEFAULT VALUE: ""
+   */
+  "extends": "../../api-extractor-shared.json",
+  "mainEntryPointFilePath": "./dist/index.d.ts"
+}

plugins/cartesia/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@livekit/agents-plugin-cartesia",
+  "version": "0.0.0",
+  "description": "Cartesia plugin for LiveKit Node Agents",
+  "main": "dist/index.js",
+  "require": "dist/index.cjs",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "author": "LiveKit",
+  "type": "module",
+  "repository": "[email protected]:livekit/agents-js.git",
+  "license": "Apache-2.0",
+  "files": [
+    "dist",
+    "src",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsup --onSuccess \"tsc --declaration --emitDeclarationOnly\"",
+    "clean": "rm -rf dist",
+    "clean:build": "pnpm clean && pnpm build",
+    "lint": "eslint -f unix \"src/**/*.{ts,js}\"",
+    "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript",
+    "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose"
+  },
+  "devDependencies": {
+    "@livekit/agents": "workspace:^x",
+    "@livekit/agents-plugin-openai": "workspace:^x",
+    "@livekit/agents-plugins-test": "workspace:^x",
+    "@livekit/rtc-node": "^0.12.1",
+    "@microsoft/api-extractor": "^7.35.0",
+    "@types/ws": "^8.5.10",
+    "tsup": "^8.3.5",
+    "typescript": "^5.0.0"
+  },
+  "dependencies": {
+    "ws": "^8.16.0"
+  },
+  "peerDependencies": {
+    "@livekit/agents": "workspace:^x",
+    "@livekit/rtc-node": "^0.12.1"
+  }
+}

plugins/cartesia/src/index.ts

@@ -0,0 +1,5 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export * from './tts.js';

plugins/cartesia/src/models.ts

@@ -0,0 +1,45 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export type TTSModels = 'sonic-english' | 'sonic-multilingual';
+
+export type TTSLanguages = 'en' | 'es' | 'fr' | 'de' | 'pt' | 'zh' | 'ja';
+
+export const TTSDefaultVoiceId = 'c2ac25f9-ecc4-4f56-9095-651354df60c0';
+
+export type TTSVoiceSpeed = 'fastest' | 'fast' | 'normal' | 'slow' | 'slowest';
+
+export type TTSVoiceEmotion =
+  | 'anger:lowest'
+  | 'anger:low'
+  | 'anger'
+  | 'anger:high'
+  | 'anger:highest'
+  | 'positivity:lowest'
+  | 'positivity:low'
+  | 'positivity'
+  | 'positivity:high'
+  | 'positivity:highest'
+  | 'surprise:lowest'
+  | 'surprise:low'
+  | 'surprise'
+  | 'surprise:high'
+  | 'surprise:highest'
+  | 'sadness:lowest'
+  | 'sadness:low'
+  | 'sadness'
+  | 'sadness:high'
+  | 'sadness:highest'
+  | 'curiosity:lowest'
+  | 'curiosity:low'
+  | 'curiosity'
+  | 'curiosity:high'
+  | 'curiosity:highest';
+
+export type TTSEncoding =
+  // XXX(nbsp): not yet supported
+  // | 'pcm_f32le'
+  // | 'pcm_mulaw'
+  // | 'pcm_alaw'
+  'pcm_s16le';

plugins/cartesia/src/tts.test.ts

@@ -0,0 +1,11 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { STT } from '@livekit/agents-plugin-openai';
+import { tts } from '@livekit/agents-plugins-test';
+import { describe } from 'vitest';
+import { TTS } from './tts.js';
+
+describe('Cartesia', async () => {
+  await tts(new TTS(), new STT());
+});