diff --git a/README_Refactor.md b/README_Refactor.md index 1baee58..dd37810 100644 --- a/README_Refactor.md +++ b/README_Refactor.md @@ -7,10 +7,43 @@ Refactor 12.23.24 - Added FileClient class and methods for file upload and download - Added SQLiteCloudVectorClient class and methods for upsert and query -Refactor 12.24.14 +Refactor Summary +- The Plan: + - Improve the usability of the SQLite Cloud platform by consolidating various features + under a single client with one consistent design and interface +The Objective: + - Provide a streamlined and consistent api for discovering, learning and using features on SQLite Cloud + - Improve the visibility of various features on the SQLite Cloud platform by providing explicit namespaces and methods for: + - functions + - file storage + - Pub/Sub + - Vector search + - Weblite (platform-level database management) + - database (core database connection) + - Increase adoption of SQLite Cloud's JS SDK by expanding our documentation. + - Provide a solid architecture for future SDK development. + - Goals: + - Streamline the onboarding and implementation process for building JS apps on SQLite Cloud + +Guidelines: + - Use consistent and scalable designs to improve readability, usability and maintainability. + Scope of work: - refactor new code to improve code smells and readability - Recap progress. + - packages + - functions: + - Purpose: used to interact with edge functions deployed on the SQLite Cloud platform + - Value: removes need for custom client + - Objective: simplify the onboarding and use of edge functions to increase adoption + - storage: + - Purpose: used to store files, with an emphasis on images and photos + - Value: unifies development experience of handling transactional and non-transactional data + - Objective: simplify the development process + - pubsub: + - Purpose: used to interact with the SQLite Cloud pubsub platform + - Value: removes need for custom client + - Objective: simplify the onboarding and use of pubsub to increase adoption - write tests for each new class - Idenitfy protential issues - Plan refactor with psuedo code @@ -21,4 +54,9 @@ TODO: - add error handling and logging - add tests - add comments -- add documentation \ No newline at end of file +- add documentation + + +Out of scope: +- Auth module (awaiting auth.js merge) +- Vector search module \ No newline at end of file diff --git a/demo.ts b/demo.ts index ca9621a..8c27f1d 100644 --- a/demo.ts +++ b/demo.ts @@ -48,7 +48,7 @@ pubSub.notifyChannel('messages', 'my message') * In the refactor, Database still exists and works as before. */ -import { createClient } from './src/SQLiteCloudClient' +import { createClient } from './src/packages/SQLiteCloudClient' const client = createClient('connection-string/chinook.db') diff --git a/package.json b/package.json index ae196c8..f63491f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@sqlitecloud/drivers", - "version": "1.0.358", + "version": "1.0.359", "description": "SQLiteCloud drivers for Typescript/Javascript in edge, web and node clients", "main": "./lib/index.js", "types": "./lib/index.d.ts", diff --git a/src/SQLiteCloudClient.ts b/src/SQLiteCloudClient.ts deleted file mode 100644 index ac47c4b..0000000 --- a/src/SQLiteCloudClient.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { Database } from './drivers/database' -import { Fetch, fetchWithAuth } from './packages/utils/fetch' -import { SQLiteCloudPubSubClient } from './packages/pubsub/SQLiteCloudPubSubClient' -import { SQLiteCloudWebliteClient } from './packages/weblite/SQLiteCloudWebliteClient' -import { SQLiteCloudFileClient } from './packages/storage/SQLiteCloudFileClient' -import { SQLiteCloudCommand } from './drivers/types' -import { getDefaultDatabase } from './packages/utils' - -interface SQLiteCloudClientConfig { - connectionString: string - fetch?: Fetch -} - -export class SQLiteCloudClient { - private connectionString: string - private fetch: Fetch - private _db: Database - - constructor(config: SQLiteCloudClientConfig | string) { - let connectionString: string - let customFetch: Fetch | undefined - - if (typeof config === 'string') { - connectionString = config - } else { - connectionString = config.connectionString - customFetch = config.fetch - } - - this.connectionString = connectionString - this.fetch = fetchWithAuth(this.connectionString, customFetch) - this.defaultDb = getDefaultDatabase(this.connectionString) ?? '' - this._db = new Database(this.connectionString) - } - - async sql(sql: TemplateStringsArray | string | SQLiteCloudCommand, ...values: any[]) { - this.db.exec(`USE DATABASE ${this.defaultDb}`) - try { - const result = await this.db.sql(sql, ...values) - return { data: result, error: null } - } catch (error) { - return { error, data: null } - } - } - - get pubSub() { - return new SQLiteCloudPubSubClient(this.db) - } - - get db() { - return this._db - } - - get weblite() { - return new SQLiteCloudWebliteClient(this.connectionString, this.fetch) - } - - get files() { - return new SQLiteCloudFileClient(this.connectionString, this.fetch) - } - - set defaultDb(dbName: string) { - this.defaultDb = dbName - } -} - -export function createClient(config: SQLiteCloudClientConfig | string): SQLiteCloudClient { - return new SQLiteCloudClient(config) -} diff --git a/src/drivers/constants.ts b/src/drivers/constants.ts index 154cd02..fa6bea9 100644 --- a/src/drivers/constants.ts +++ b/src/drivers/constants.ts @@ -11,7 +11,10 @@ if (typeof Deno !== 'undefined') { JS_ENV = 'node' } -export const DEFAULT_HEADERS = { 'X-Client-Info': `sqlitecloud-js-${JS_ENV}/${version}` } +export const DEFAULT_HEADERS = { + 'X-Client-Info': `sqlitecloud-js-${JS_ENV}/${version}`, + 'Content-Type': 'application/octet-stream' +} export const DEFAULT_GLOBAL_OPTIONS = { headers: DEFAULT_HEADERS } diff --git a/src/drivers/types.ts b/src/drivers/types.ts index e5a12d4..7d8cdb4 100644 --- a/src/drivers/types.ts +++ b/src/drivers/types.ts @@ -163,4 +163,5 @@ export enum SQLiteCloudArrayType { export type UploadOptions = { replace?: boolean + headers?: Record } diff --git a/src/packages/SQLiteCloudClient.ts b/src/packages/SQLiteCloudClient.ts new file mode 100644 index 0000000..1c8c91d --- /dev/null +++ b/src/packages/SQLiteCloudClient.ts @@ -0,0 +1,81 @@ +import { Database } from '../drivers/database' +import { Fetch, fetchWithAuth } from './utils/fetch' +import { PubSubClient } from './pubsub/PubSubClient' +import { SQLiteCloudWebliteClient } from './weblite/SQLiteCloudWebliteClient' +import { StorageClient } from './storage/SQLiteCloudStorageClient' +import { SQLiteCloudCommand, SQLiteCloudError } from '../drivers/types' +import { cleanConnectionString, getDefaultDatabase } from './utils' + +interface SQLiteCloudClientConfig { + connectionString: string + fetch?: Fetch +} + +export class SQLiteCloudClient { + protected connectionString: string + protected fetch: Fetch + protected _db: Database + + constructor(config: SQLiteCloudClientConfig | string) { + try { + if (!config) { + throw new SQLiteCloudError('Invalid connection string or config') + } + let connectionString: string + let customFetch: Fetch | undefined + + if (typeof config === 'string') { + connectionString = cleanConnectionString(config) + } else { + connectionString = config.connectionString + customFetch = config.fetch + } + + this.connectionString = connectionString + this.fetch = fetchWithAuth(this.connectionString, customFetch) + this.defaultDb = getDefaultDatabase(this.connectionString) ?? '' + this._db = new Database(this.connectionString) + } catch (error) { + throw new SQLiteCloudError('failed to initialize SQLiteCloudClient') + } + } + + async sql(sql: TemplateStringsArray | string | SQLiteCloudCommand, ...values: any[]) { + this.db.exec(`USE DATABASE ${this.defaultDb}`) + try { + const result = await this.db.sql(sql, ...values) + return { data: result, error: null } + } catch (error) { + return { error, data: null } + } + } + + get pubSub() { + return new PubSubClient(this.db.getConfiguration()) + } + + get db() { + return this._db + } + + get weblite() { + return new SQLiteCloudWebliteClient(this.connectionString, this.fetch) + } + + get files() { + return new StorageClient(this.connectionString, this.fetch) + } + + get functions() { + // return new SQLiteCloudFunctionsClient(this.connectionString, this.fetch) + return null + } + + set defaultDb(dbName: string) { + this.defaultDb = dbName + } +} + +export function createClient(config: SQLiteCloudClientConfig | string): SQLiteCloudClient { + return new SQLiteCloudClient(config) +} diff --git a/src/packages/constants/index.ts b/src/packages/constants/index.ts new file mode 100644 index 0000000..402e127 --- /dev/null +++ b/src/packages/constants/index.ts @@ -0,0 +1,2 @@ +export const FILES_DATABASE = 'files.sqlite' +export const FUNCTIONS_ROOT_PATH = 'functions' \ No newline at end of file diff --git a/src/packages/functions/FunctionsClient.ts b/src/packages/functions/FunctionsClient.ts new file mode 100644 index 0000000..973e2b3 --- /dev/null +++ b/src/packages/functions/FunctionsClient.ts @@ -0,0 +1,36 @@ +import { SQLiteCloudError } from '../../drivers/types' +import { FUNCTIONS_ROOT_PATH } from '../constants' +import { getAPIUrl } from '../utils' +import { Fetch, resolveFetch, resolveHeadersConstructor } from '../utils/fetch' + +export class FunctionsClient { + protected url: string + protected fetch: Fetch + protected headers: Record + + constructor( + connectionString: string, + options: { + customFetch?: Fetch, + headers?: Record + } = {} + ) { + this.url = getAPIUrl(connectionString, FUNCTIONS_ROOT_PATH) + this.fetch = resolveFetch(options.customFetch) + this.headers = options.headers ?? {} + } + // auth token is the full connection string with apikey + setAuth(token: string) { + this.headers.Authorization = `Bearer ${token}` + } + + async invoke(functionName: string, args: any[]) { + try { + // TODO IMPLEMENT + } catch (error) { + throw new SQLiteCloudError(`Failed to invoke function: ${error}`) + } + } + + +} diff --git a/src/packages/pubsub/SQLiteCloudPubSubClient.ts b/src/packages/pubsub/PubSubClient.ts similarity index 75% rename from src/packages/pubsub/SQLiteCloudPubSubClient.ts rename to src/packages/pubsub/PubSubClient.ts index 6ecb180..e8cdda7 100644 --- a/src/packages/pubsub/SQLiteCloudPubSubClient.ts +++ b/src/packages/pubsub/PubSubClient.ts @@ -1,7 +1,6 @@ import { SQLiteCloudConnection } from '../../drivers/connection' import SQLiteCloudTlsConnection from '../../drivers/connection-tls' -import { Database } from '../../drivers/database' -import { getDbFromConfig } from '../utils' +import { SQLiteCloudConfig } from '../../drivers/types' export type PubSubCallback = (error: Error | null, results?: T) => void @@ -10,7 +9,7 @@ export interface ListenOptions { dbName?: string } -interface SQLiteCloudPubSub { +export interface PubSub { listen(options: ListenOptions, callback: PubSubCallback): Promise unlisten(options: ListenOptions): void subscribe(channelName: string, callback: PubSubCallback): Promise @@ -26,17 +25,17 @@ interface SQLiteCloudPubSub { /** * Pub/Sub class to receive changes on database tables or to send messages to channels. */ -export class SQLiteCloudPubSubClient implements SQLiteCloudPubSub { +export class PubSubClient implements PubSub { + protected _pubSubConnection: SQLiteCloudConnection | null + protected defaultDatabaseName: string + protected config: SQLiteCloudConfig // instantiate in createConnection? - constructor(queryConnection: Database) { - this.queryConnection = queryConnection + constructor(config: SQLiteCloudConfig) { + this.config = config this._pubSubConnection = null - this.defaultDatabaseName = getDbFromConfig(queryConnection.getConfiguration()) + this.defaultDatabaseName = config?.database ?? '' } - private queryConnection: Database - private _pubSubConnection: SQLiteCloudConnection | null - private defaultDatabaseName: string /** * Listen to a channel and start to receive messages to the provided callback. * @param options Options for the listen operation. If tablename and channelName are provided, channelName is used. @@ -44,16 +43,16 @@ export class SQLiteCloudPubSubClient implements SQLiteCloudPubSub { * @param callback Callback to be called when a message is received */ - private get pubSubConnection(): SQLiteCloudConnection { + get pubSubConnection(): SQLiteCloudConnection { if (!this._pubSubConnection) { - this._pubSubConnection = new SQLiteCloudTlsConnection(this.queryConnection.getConfiguration()) + this._pubSubConnection = new SQLiteCloudTlsConnection(this.config) } return this._pubSubConnection } - public async listen(options: ListenOptions, callback: PubSubCallback): Promise { + async listen(options: ListenOptions, callback: PubSubCallback): Promise { const _dbName = options.dbName ? options.dbName : this.defaultDatabaseName; - const authCommand: string = await this.queryConnection.sql`LISTEN ${options.tableName} DATABASE ${_dbName};` + const authCommand: string = await this.pubSubConnection.sql`LISTEN ${options.tableName} DATABASE ${_dbName};` return new Promise((resolve, reject) => { this.pubSubConnection.sendCommands(authCommand, (error, results) => { @@ -77,11 +76,11 @@ export class SQLiteCloudPubSubClient implements SQLiteCloudPubSub { * @param entityName Name of the table or the channel */ public unlisten(options: ListenOptions): void { - this.queryConnection.sql`UNLISTEN ${options.tableName} DATABASE ${options.dbName};` + this.pubSubConnection.sql`UNLISTEN ${options.tableName} DATABASE ${options.dbName};` } public async subscribe(channelName: string, callback: PubSubCallback): Promise { - const authCommand: string = await this.queryConnection.sql`LISTEN ${channelName};` + const authCommand: string = await this.pubSubConnection.sql`LISTEN ${channelName};` return new Promise((resolve, reject) => { this.pubSubConnection.sendCommands(authCommand, (error, results) => { @@ -96,7 +95,7 @@ export class SQLiteCloudPubSubClient implements SQLiteCloudPubSub { } public unsubscribe(channelName: string): void { - this.queryConnection.sql`UNLISTEN ${channelName};` + this.pubSubConnection.sql`UNLISTEN ${channelName};` } /** @@ -106,7 +105,7 @@ export class SQLiteCloudPubSubClient implements SQLiteCloudPubSub { */ public async create(channelName: string, failIfExists: boolean = true): Promise { // type this output - return await this.queryConnection.sql(`CREATE CHANNEL ?${failIfExists ? '' : ' IF NOT EXISTS'};`, channelName) + return await this.pubSubConnection.sql(`CREATE CHANNEL ?${failIfExists ? '' : ' IF NOT EXISTS'};`, channelName) } /** @@ -115,7 +114,7 @@ export class SQLiteCloudPubSubClient implements SQLiteCloudPubSub { */ public async delete(channelName: string): Promise { // type this output - return await this.queryConnection.sql(`REMOVE CHANNEL ?;`, channelName) + return await this.pubSubConnection.sql(`REMOVE CHANNEL ?;`, channelName) } /** @@ -123,7 +122,7 @@ export class SQLiteCloudPubSubClient implements SQLiteCloudPubSub { */ public notify(channelName: string, message: string): Promise { // type this output - return this.queryConnection.sql`NOTIFY ${channelName} ${message};` + return this.pubSubConnection.sql`NOTIFY ${channelName} ${message};` } /** @@ -137,7 +136,7 @@ export class SQLiteCloudPubSubClient implements SQLiteCloudPubSub { if (error) { reject(error) } else { - this.queryConnection.close() + this.pubSubConnection.close() resolve(results) } }) diff --git a/src/packages/storage/SQLiteCloudFileClient.ts b/src/packages/storage/SQLiteCloudStorageClient.ts similarity index 90% rename from src/packages/storage/SQLiteCloudFileClient.ts rename to src/packages/storage/SQLiteCloudStorageClient.ts index fd63505..033eb0c 100644 --- a/src/packages/storage/SQLiteCloudFileClient.ts +++ b/src/packages/storage/SQLiteCloudStorageClient.ts @@ -2,20 +2,18 @@ import { SQLiteCloudError } from "../../drivers/types" import { getAPIUrl } from "../utils" import { Fetch, fetchWithAuth } from "../utils/fetch" -interface SQLiteCloudFile { - createBucket(bucket: string, path: string): Promise +interface Storage { + createBucket(bucket: string): Promise getBucket(bucket: string): Promise - deleteBucket(bucket: string): Promise + deleteBucket(bucket: string): Promise listBuckets(): Promise - upload(bucket: string, pathname: string, file: File | Buffer | Blob | string, options: { contentType: string }): Promise + upload(bucket: string, pathname: string, file: File | Buffer | Blob | string, options: { contentType: string }): Promise download(bucket: string, pathname: string): Promise - remove(bucket: string, pathName: string): Promise + remove(bucket: string, pathName: string): Promise list(bucket: string): Promise } -const FILES_DATABASE = 'files.sqlite' - -export class SQLiteCloudFileClient implements SQLiteCloudFile { +export class StorageClient implements Storage { private filesUrl: string private webliteSQLUrl: string private fetch: Fetch @@ -102,4 +100,3 @@ export class SQLiteCloudFileClient implements SQLiteCloudFile { return response.json() } } - diff --git a/src/packages/utils/index.ts b/src/packages/utils/index.ts index 3b976ab..4d3e45f 100644 --- a/src/packages/utils/index.ts +++ b/src/packages/utils/index.ts @@ -23,3 +23,32 @@ export const getDefaultDatabase = (connectionString: string) => { export const getDbFromConfig = (config: SQLiteCloudConfig) => new URL(config.connectionstring ?? '')?.pathname.split('/').pop() ?? '' export const formatCommand = (arr: string[]) => arr.reduce((acc, curr) => curr.length > 0 ? (acc + ' ' + curr) : acc, '') + ';' + +/** + * Cleans and validates the SQLite Cloud connection string + * @param connectionString - The connection string to clean + * @returns The cleaned connection string + * @throws Error if connection string is invalid + * + * @example + * ```ts + * // Valid connection string + * cleanConnectionString('sqlitecloud://username:password@host:port/database') + * + * // Removes trailing slash + * cleanConnectionString('sqlitecloud://username:password@host:port/database/') + * + * // Throws error + * cleanConnectionString('invalid-connection-string') + * ``` + */ + +export const cleanConnectionString = (connectionString: string) => { + if (!connectionString.includes('sqlitecloud://')) { + throw new Error('Invalid connection string') + } + if (connectionString.endsWith('/')) { + return connectionString.slice(0, -1) + } + return connectionString +} \ No newline at end of file diff --git a/src/packages/weblite/SQLiteCloudWebliteClient.ts b/src/packages/weblite/SQLiteCloudWebliteClient.ts index 4a163d0..30eed0c 100644 --- a/src/packages/weblite/SQLiteCloudWebliteClient.ts +++ b/src/packages/weblite/SQLiteCloudWebliteClient.ts @@ -3,12 +3,16 @@ import { Fetch, fetchWithAuth } from '../utils/fetch' import { DEFAULT_HEADERS } from '../../drivers/constants' import { getAPIUrl } from '../utils' +interface WebliteResponse { + data: any, + error: SQLiteCloudError | null +} interface SQLiteCloudWeblite { - upload(databaseName: string, file: File | Buffer | Blob | string, opts: UploadOptions): Promise - download(databaseName: string): Promise - delete(databaseName: string): Promise - listDatabases(): Promise - create(databaseName: string): Promise + upload(dbName: string, file: File | Buffer | Blob | string, opts: UploadOptions): Promise + download(dbName: string): Promise + delete(dbName: string): Promise + listDatabases(): Promise + create(dbName: string): Promise } export class SQLiteCloudWebliteClient implements SQLiteCloudWeblite { @@ -20,70 +24,79 @@ export class SQLiteCloudWebliteClient implements SQLiteCloudWeblite { this.fetch = fetch || fetchWithAuth(connectionString) } - async upload(databaseName: string, file: File | Buffer | Blob | string, opts: UploadOptions = {}) { - const url = `${this.webliteUrl}/${databaseName}` - let body - if (file instanceof File) { - body = file - } else if (file instanceof Buffer) { - body = file - } else if (file instanceof Blob) { - body = file - } else { - // string - body = new Blob([file]) - } - - const headers = { - 'Content-Type': 'application/octet-stream', - 'X-Client-Info': DEFAULT_HEADERS['X-Client-Info'] - } - - const method = opts.replace ? 'PATCH' : 'POST' + async upload( + dbName: string, + file: File | Buffer | Blob | string, + opts: UploadOptions = {} + ) { + const url = `${this.webliteUrl}/${dbName}` + let body: File | Buffer | Blob | string + let headers = {} + if (file instanceof File) { + body = file - const response = await this.fetch(url, { method, body, headers }) + } else if (file instanceof Buffer) { + body = file + } else if (file instanceof Blob) { + body = file + } else { + // string + body = new Blob([file]) + } + + headers = { + ...(opts.headers ?? {}), + ...headers, + ...DEFAULT_HEADERS, + } + + const method = opts.replace ? 'PATCH' : 'POST' + const response = await this.fetch(url, { method, body, headers }) if (!response.ok) { - throw new SQLiteCloudError(`Failed to upload database: ${response.statusText}`) + return { data: null, error: new SQLiteCloudError(`Failed to upload database: ${response.statusText}`) } } - return response + const data = await response.json() + + return { data, error: null } } - async download(databaseName: string) { - const url = `${this.webliteUrl}/${databaseName}` + async download(dbName: string) { + const url = `${this.webliteUrl}/${dbName}` const response = await this.fetch(url, { method: 'GET' }) if (!response.ok) { - throw new SQLiteCloudError(`Failed to download database: ${response.statusText}`) + return { data: null, error: new SQLiteCloudError(`Failed to download database: ${response.statusText}`) } } const isNode = typeof window === 'undefined' - return isNode ? await response.arrayBuffer() : await response.blob() + const data = isNode ? await response.arrayBuffer() : await response.blob() + return { data, error: null } } - async delete(databaseName: string) { - const url = `${this.webliteUrl}/${databaseName}` + async delete(dbName: string) { + const url = `${this.webliteUrl}/${dbName}` const response = await this.fetch(url, { method: 'DELETE' }) if (!response.ok) { - throw new SQLiteCloudError(`Failed to delete database: ${response.statusText}`) + return { data: null, error: new SQLiteCloudError(`Failed to delete database: ${response.statusText}`) } } - return response + return { data: null, error: null } } async listDatabases() { const url = `${this.webliteUrl}/databases` const response = await this.fetch(url, { method: 'GET' }) if (!response.ok) { - throw new SQLiteCloudError(`Failed to list databases: ${response.statusText}`) + return { data: null, error: new SQLiteCloudError(`Failed to list databases: ${response.statusText}`) } } - return await response.json() + return { data: await response.json(), error: null } } - async create(databaseName: string) { - const response = await fetch(`${this.webliteUrl}/sql?sql=CREATE DATABASE ${databaseName}`, { method: 'POST' }) + async create(dbName: string) { + const response = await fetch(`${this.webliteUrl}/sql?sql=CREATE DATABASE ${dbName}`, { method: 'POST' }) if (!response.ok) { - throw new SQLiteCloudError(`Failed to create database: ${response.statusText}`) + return { data: null, error: new SQLiteCloudError(`Failed to create database: ${response.statusText}`) } } - return response + return { data: null, error: null } } } diff --git a/src/packages/test/SQLiteCloudClient.test.ts b/test/SQLiteCloudClient.test.ts similarity index 100% rename from src/packages/test/SQLiteCloudClient.test.ts rename to test/SQLiteCloudClient.test.ts