From d3904cdd473193d72cb2465ce79019e718631e6c Mon Sep 17 00:00:00 2001 From: Jacob Prall Date: Tue, 24 Dec 2024 23:36:14 -0800 Subject: [PATCH] improve client library, apis and error handling --- src/packages/SQLiteCloudClient.ts | 8 +- src/packages/functions/FunctionsClient.ts | 61 +++++- src/packages/pubsub/PubSubClient.ts | 2 + .../storage/SQLiteCloudStorageClient.ts | 102 ---------- src/packages/storage/StorageClient.ts | 175 ++++++++++++++++++ .../vector/SQLiteCloudVectorClient.ts | 9 +- .../weblite/SQLiteCloudWebliteClient.ts | 22 ++- 7 files changed, 255 insertions(+), 124 deletions(-) delete mode 100644 src/packages/storage/SQLiteCloudStorageClient.ts create mode 100644 src/packages/storage/StorageClient.ts diff --git a/src/packages/SQLiteCloudClient.ts b/src/packages/SQLiteCloudClient.ts index 1c8c91d..3ed35bf 100644 --- a/src/packages/SQLiteCloudClient.ts +++ b/src/packages/SQLiteCloudClient.ts @@ -1,8 +1,8 @@ import { Database } from '../drivers/database' import { Fetch, fetchWithAuth } from './utils/fetch' import { PubSubClient } from './pubsub/PubSubClient' -import { SQLiteCloudWebliteClient } from './weblite/SQLiteCloudWebliteClient' -import { StorageClient } from './storage/SQLiteCloudStorageClient' +import { WebliteClient } from './weblite/SQLiteCloudWebliteClient' +import { StorageClient } from './storage/StorageClient' import { SQLiteCloudCommand, SQLiteCloudError } from '../drivers/types' import { cleanConnectionString, getDefaultDatabase } from './utils' @@ -59,11 +59,11 @@ export class SQLiteCloudClient { } get weblite() { - return new SQLiteCloudWebliteClient(this.connectionString, this.fetch) + return new WebliteClient(this.connectionString, { customFetch: this.fetch }) } get files() { - return new StorageClient(this.connectionString, this.fetch) + return new StorageClient(this.connectionString, { customFetch: this.fetch }) } get functions() { diff --git a/src/packages/functions/FunctionsClient.ts b/src/packages/functions/FunctionsClient.ts index 973e2b3..486e800 100644 --- a/src/packages/functions/FunctionsClient.ts +++ b/src/packages/functions/FunctionsClient.ts @@ -1,3 +1,4 @@ +import { DEFAULT_HEADERS } from '../../drivers/constants' import { SQLiteCloudError } from '../../drivers/types' import { FUNCTIONS_ROOT_PATH } from '../constants' import { getAPIUrl } from '../utils' @@ -17,20 +18,68 @@ export class FunctionsClient { ) { this.url = getAPIUrl(connectionString, FUNCTIONS_ROOT_PATH) this.fetch = resolveFetch(options.customFetch) - this.headers = options.headers ?? {} + this.headers = options.headers ? { ...DEFAULT_HEADERS, ...options.headers } : { ...DEFAULT_HEADERS } } // auth token is the full connection string with apikey setAuth(token: string) { this.headers.Authorization = `Bearer ${token}` } - async invoke(functionName: string, args: any[]) { + async invoke(functionId: string, args: any[]) { try { - // TODO IMPLEMENT + const response = await this.fetch(`${this.url}/${functionId}`, { + method: 'POST', + body: JSON.stringify(args), + headers: this.headers + }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to invoke function: ${response.statusText}`) + } + let responseType = (response.headers.get('Content-Type') ?? 'text/plain').split(';')[0].trim() + let data: any + if (responseType === 'application/json') { + data = await response.json() + } else if (responseType === 'application/octet-stream') { + data = await response.blob() + } else if (responseType === 'text/event-stream') { + data = response + } else if (responseType === 'multipart/form-data') { + data = await response.formData() + } else { + // default to text + data = await response.text() + } + return { data, error: null } } catch (error) { - throw new SQLiteCloudError(`Failed to invoke function: ${error}`) + return { data: null, error } } } - - } + +/** + if ( + functionArgs && + ((headers && !Object.prototype.hasOwnProperty.call(headers, 'Content-Type')) || !headers) + ) { + if ( + (typeof Blob !== 'undefined' && functionArgs instanceof Blob) || + functionArgs instanceof ArrayBuffer + ) { + // will work for File as File inherits Blob + // also works for ArrayBuffer as it is the same underlying structure as a Blob + _headers['Content-Type'] = 'application/octet-stream' + body = functionArgs + } else if (typeof functionArgs === 'string') { + // plain string + _headers['Content-Type'] = 'text/plain' + body = functionArgs + } else if (typeof FormData !== 'undefined' && functionArgs instanceof FormData) { + // don't set content-type headers + // Request will automatically add the right boundary value + body = functionArgs + } else { + // default, assume this is JSON + _headers['Content-Type'] = 'application/json' + body = JSON.stringify(functionArgs) + } + */ \ No newline at end of file diff --git a/src/packages/pubsub/PubSubClient.ts b/src/packages/pubsub/PubSubClient.ts index e8cdda7..500e88e 100644 --- a/src/packages/pubsub/PubSubClient.ts +++ b/src/packages/pubsub/PubSubClient.ts @@ -125,6 +125,8 @@ export class PubSubClient implements PubSub { return this.pubSubConnection.sql`NOTIFY ${channelName} ${message};` } + // DOUBLE CHECK THIS + /** * Ask the server to close the connection to the database and * to keep only open the Pub/Sub connection. diff --git a/src/packages/storage/SQLiteCloudStorageClient.ts b/src/packages/storage/SQLiteCloudStorageClient.ts deleted file mode 100644 index 033eb0c..0000000 --- a/src/packages/storage/SQLiteCloudStorageClient.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { SQLiteCloudError } from "../../drivers/types" -import { getAPIUrl } from "../utils" -import { Fetch, fetchWithAuth } from "../utils/fetch" - -interface Storage { - createBucket(bucket: string): Promise - getBucket(bucket: string): Promise - deleteBucket(bucket: string): Promise - listBuckets(): Promise - upload(bucket: string, pathname: string, file: File | Buffer | Blob | string, options: { contentType: string }): Promise - download(bucket: string, pathname: string): Promise - remove(bucket: string, pathName: string): Promise - list(bucket: string): Promise -} - -export class StorageClient implements Storage { - private filesUrl: string - private webliteSQLUrl: string - private fetch: Fetch - - constructor(connectionString: string, sql?: Fetch) { - this.filesUrl = getAPIUrl(connectionString, 'files') - this.webliteSQLUrl = getAPIUrl(connectionString, 'weblite/sql') - this.fetch = fetchWithAuth(connectionString, fetch) - } - - async createBucket(bucket: string) { - const url = `${this.webliteSQLUrl}?sql=USE DATABASE files; INSERT INTO files (Bucket) VALUES ('${bucket}');` - const response = await this.fetch(url, { method: 'POST' }) - if (!response.ok) { - throw new SQLiteCloudError(`Failed to create bucket: ${response.statusText}`) - } - return response.json() - } - - async getBucket(bucket: string) { - const url = `${this.filesUrl}/${bucket}` - const response = await this.fetch(url, { method: 'GET' }) - if (!response.ok) { - throw new SQLiteCloudError(`Failed to get bucket: ${response.statusText}`) - } - - return response.json() - } - - async deleteBucket(bucket: string) { - const url = `${this.filesUrl}/${bucket}` - const response = await this.fetch(url, { method: 'DELETE' }) - if (!response.ok) { - throw new SQLiteCloudError(`Failed to delete bucket: ${response.statusText}`) - } - return response.json() - } - - async listBuckets() { - const encodedUrl = encodeURIComponent(`${this.webliteSQLUrl}?sql=USE DATABASE files.sqlite; SELECT * FROM files`) - const response = await this.fetch(encodedUrl) - if (!response.ok) { - throw new SQLiteCloudError(`Failed to list buckets: ${response.statusText}`) - } - return response.json() - } - - async upload(bucket: string, pathname: string, file: File | Buffer | Blob | string, options: { contentType: string }) { - const url = `${this.filesUrl}/${bucket}/${pathname}`; - const headers = { - 'Content-Type': options?.contentType || 'application/octet-stream' - } - const response = await this.fetch(url, { method: 'POST', body: file, headers }) - if (!response.ok) { - throw new SQLiteCloudError(`Failed to upload file: ${response.statusText}`) - } - return response.json() - } - - async download(bucket: string, pathname: string) { - const url = `${this.filesUrl}/${bucket}/${pathname}`; - const response = await this.fetch(url, { method: 'GET' }) - if (!response.ok) { - throw new SQLiteCloudError(`Failed to download file: ${response.statusText}`) - } - return response.blob() - } - - async remove(bucket: string, pathName: string) { - const url = `${this.filesUrl}/${bucket}/${pathName}` - const response = await this.fetch(url, { method: 'DELETE' }) - if (!response.ok) { - throw new SQLiteCloudError(`Failed to remove file: ${response.statusText}`) - } - return response.json() - } - - async list(bucket: string) { - const encodedUrl = encodeURIComponent(`${this.webliteSQLUrl}?sql=USE DATABASE files.sqlite; SELECT * FROM files WHERE bucket = '${bucket}'`) - const response = await this.fetch(encodedUrl) - if (!response.ok) { - throw new SQLiteCloudError(`Failed to list files: ${response.statusText}`) - } - return response.json() - } -} diff --git a/src/packages/storage/StorageClient.ts b/src/packages/storage/StorageClient.ts new file mode 100644 index 0000000..bb6156b --- /dev/null +++ b/src/packages/storage/StorageClient.ts @@ -0,0 +1,175 @@ +import { DEFAULT_HEADERS } from "../../drivers/constants" +import { SQLiteCloudError } from "../../drivers/types" +import { getAPIUrl } from "../utils" +import { Fetch, fetchWithAuth } from "../utils/fetch" + +interface StorageResponse { + data: any + error: any +} + +interface Storage { + createBucket(bucket: string): Promise + getBucket(bucket: string): Promise + deleteBucket(bucket: string): Promise + listBuckets(): Promise + upload(bucket: string, pathname: string, file: File | Buffer | Blob | string, options: { contentType: string }): Promise + download(bucket: string, pathname: string): Promise + remove(bucket: string, pathName: string): Promise + list(bucket: string): Promise +} + +export class StorageClient implements Storage { + protected filesUrl: string + protected webliteSQLUrl: string + protected headers: Record + protected fetch: Fetch + + constructor( + connectionString: string, + options: { + customFetch?: Fetch, + headers?: Record + } = {}) { + this.filesUrl = getAPIUrl(connectionString, 'files') + this.webliteSQLUrl = getAPIUrl(connectionString, 'weblite/sql') + this.fetch = options.customFetch || fetchWithAuth(connectionString) + this.headers = options.headers ? { ...DEFAULT_HEADERS, ...options.headers } : { ...DEFAULT_HEADERS } + } + + async createBucket(bucket: string) { + const sql = `USE DATABASE files; INSERT INTO files (Bucket) VALUES ('${bucket}');` + + try { + const response = await this.fetch(this.webliteSQLUrl, { + method: 'POST', + body: JSON.stringify({ sql }), + headers: this.headers + }) + + if (!response.ok) { + throw new SQLiteCloudError(`Failed to create bucket: ${response.statusText}`) + } + + return { data: await response.json(), error: null } + } catch (error) { + return { data: null, error } + } + } + + async getBucket(bucket: string) { + const url = `${this.filesUrl}/${bucket}` + const response = await this.fetch(url, { method: 'GET', headers: this.headers }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to get bucket: ${response.statusText}`) + } + + return { data: await response.json(), error: null } + } + + async deleteBucket(bucket: string) { + const url = `${this.filesUrl}/${bucket}` + try { + const response = await this.fetch(url, { method: 'DELETE', headers: this.headers }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to delete bucket: ${response.statusText}`) + } + return { data: await response.json(), error: null } + } catch (error) { + return { data: null, error } + } + } + + async listBuckets() { + const sql = `USE DATABASE files.sqlite; SELECT * FROM files;` + try { + const response = await this.fetch(this.webliteSQLUrl, { + method: 'POST', + body: JSON.stringify({ sql }), + headers: this.headers + }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to list buckets: ${response.statusText}`) + } + return { data: await response.json(), error: null } + } catch (error) { + return { + data: null, + error + } + } + } + + async upload(bucket: string, pathname: string, file: File | Buffer | Blob | string, options: { contentType: string }) { + const url = `${this.filesUrl}/${bucket}/${pathname}`; + const headers = { + 'Content-Type': options?.contentType || 'application/octet-stream' + } + try { + const response = await this.fetch(url, { method: 'POST', body: file, headers }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to upload file: ${response.statusText}`) + } + return { data: await response.json(), error: null } + } catch (error) { + return { data: null, error } + } + } + + async download(bucket: string, pathname: string) { + const url = `${this.filesUrl}/${bucket}/${pathname}`; + try { + const response = await this.fetch(url, { method: 'GET' }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to download file: ${response.statusText}`) + } + let responseType = (response.headers.get('Content-Type') ?? 'text/plain').split(';')[0].trim() + let data: any + if (responseType === 'application/json') { + data = await response.json() + } else if (responseType === 'application/octet-stream') { + data = await response.blob() + } else if (responseType === 'text/event-stream') { + data = response + } else if (responseType === 'multipart/form-data') { + data = await response.formData() + } else { + // default to text + data = await response.text() + } + return { data, error: null } + } catch (error) { + return { data: null, error } + } + } + + async remove(bucket: string, pathName: string) { + const url = `${this.filesUrl}/${bucket}/${pathName}` + try { + const response = await this.fetch(url, { method: 'DELETE' }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to remove file: ${response.statusText}`) + } + return { data: response.json(), error: null } + } catch (error) { + return { data: null, error } + } + } + + async list(bucket: string) { + const sql = `USE DATABASE files.sqlite; SELECT * FROM files WHERE bucket = '${bucket}'` + try { + const response = await this.fetch(this.webliteSQLUrl, { + method: 'POST', + body: JSON.stringify({ sql }), + headers: this.headers + }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to list files: ${response.statusText}`) + } + return { data: await response.json(), error: null } + } catch (error) { + return { data: null, error } + } + } +} diff --git a/src/packages/vector/SQLiteCloudVectorClient.ts b/src/packages/vector/SQLiteCloudVectorClient.ts index d56f80e..449d9b2 100644 --- a/src/packages/vector/SQLiteCloudVectorClient.ts +++ b/src/packages/vector/SQLiteCloudVectorClient.ts @@ -22,9 +22,9 @@ interface QueryOptions { where?: string[] } -interface SQLiteCloudVector { - init(options: IndexOptions): Promise - upsert(data: UpsertData): Promise +interface Vector { + init(options: IndexOptions): Promise + upsert(data: UpsertData): Promise query(queryEmbedding: number[], options: QueryOptions): Promise } @@ -52,8 +52,7 @@ function formatUpsertCommand(data: UpsertData): [any, any] { } -export class SQLiteCloudVectorClient implements SQLiteCloudVector { - +export class VectorClient implements Vector { private _db: Database private _tableName: string private _columns: Column[] diff --git a/src/packages/weblite/SQLiteCloudWebliteClient.ts b/src/packages/weblite/SQLiteCloudWebliteClient.ts index 30eed0c..0e8a0bb 100644 --- a/src/packages/weblite/SQLiteCloudWebliteClient.ts +++ b/src/packages/weblite/SQLiteCloudWebliteClient.ts @@ -4,10 +4,10 @@ import { DEFAULT_HEADERS } from '../../drivers/constants' import { getAPIUrl } from '../utils' interface WebliteResponse { - data: any, + data: any, // TODO: type this error: SQLiteCloudError | null } -interface SQLiteCloudWeblite { +interface Weblite { upload(dbName: string, file: File | Buffer | Blob | string, opts: UploadOptions): Promise download(dbName: string): Promise delete(dbName: string): Promise @@ -15,13 +15,21 @@ interface SQLiteCloudWeblite { create(dbName: string): Promise } -export class SQLiteCloudWebliteClient implements SQLiteCloudWeblite { - private webliteUrl: string - private fetch: Fetch +export class WebliteClient implements Weblite { + protected webliteUrl: string + protected headers: Record + protected fetch: Fetch - constructor(connectionString: string, fetch?: Fetch) { + constructor( + connectionString: string, + options: { + customFetch?: Fetch, + headers?: Record + } = {} + ) { this.webliteUrl = getAPIUrl(connectionString, 'weblite') - this.fetch = fetch || fetchWithAuth(connectionString) + this.fetch = options?.customFetch || fetchWithAuth(connectionString) + this.headers = options.headers ? { ...DEFAULT_HEADERS, ...options.headers } : { ...DEFAULT_HEADERS } } async upload(