From f4bd4e35c4308d9bb12707914ed82d9633f02e9c Mon Sep 17 00:00:00 2001 From: Mughees Khan Date: Thu, 28 Nov 2024 10:37:18 +0200 Subject: [PATCH] Update sqlite3 to v2.5.0 (#16) * Adding uuid.c example * Expand readme * Link custom extension example * Fix extension tests in CI * Add a DynamicBuffer for faster VFS implementations. * Switch to Uint8Buffer. * Support 16KiB page sizes on Android * Avoid copying for writes to memory * Changelog entry * Re-compile sqlite3 for CI * Update sqlite3 in wasm build * Implement remote file system * Fix web tests * Prepare to publish sqlite3 web * Upgrade lints * Update sqlite to 3.47.0 * Raise version number * Support custom VFS on native platforms * Reformat headers * Mark ffi vfs test as using ffi * Update simulator for iOS integration tests * IndexedDb: Store blobs as array buffers if necessary * Add changelog entry * Add sqlite3_test utility * Mention sqlite test in top-level readme * Prepare 2.5.0 release * Fixed bug when setting update_hook to null twice (#269) --------- Co-authored-by: Rody Davis Co-authored-by: Simon Binder Co-authored-by: Ralf Kistner Co-authored-by: Dominic Jack --- .github/workflows/main.yml | 12 +- README.md | 6 +- integration_tests/flutter_libs/.metadata | 30 +- .../flutter_libs/android/.gitignore | 4 +- .../flutter_libs/android/app/build.gradle | 62 ++-- .../android/app/src/debug/AndroidManifest.xml | 6 +- .../android/app/src/main/AndroidManifest.xml | 32 +- .../com/example/flutter_libs/MainActivity.kt | 3 +- .../app/src/main/res/values/styles.xml | 12 +- .../app/src/profile/AndroidManifest.xml | 6 +- .../flutter_libs/android/build.gradle | 19 +- .../flutter_libs/android/gradle.properties | 3 +- .../gradle/wrapper/gradle-wrapper.properties | 3 +- .../flutter_libs/android/settings.gradle | 32 +- integration_tests/flutter_libs/pubspec.yaml | 2 +- sqlite3/CHANGELOG.md | 15 +- sqlite3/assets/sqlite3.h | 78 +++++ sqlite3/assets/wasm/CMakeLists.txt | 3 +- sqlite3/lib/common.dart | 3 +- sqlite3/lib/open.dart | 2 +- sqlite3/lib/sqlite3.dart | 2 +- sqlite3/lib/src/ffi/api.dart | 2 +- sqlite3/lib/src/ffi/bindings.dart | 317 ++++++++++++++++-- sqlite3/lib/src/ffi/implementation.dart | 4 +- sqlite3/lib/src/ffi/memory.dart | 3 +- sqlite3/lib/src/ffi/sqlite3.g.dart | 234 +++++++++++++ sqlite3/lib/src/implementation/bindings.dart | 6 +- sqlite3/lib/src/implementation/sqlite3.dart | 16 +- .../vfs/memory.dart => in_memory_vfs.dart} | 9 +- sqlite3/lib/src/sqlite3.dart | 24 +- sqlite3/lib/src/{wasm/vfs => }/utils.dart | 0 sqlite3/lib/src/vfs.dart | 3 - sqlite3/lib/src/wasm/bindings.dart | 2 + sqlite3/lib/src/wasm/sqlite3.dart | 24 -- .../lib/src/wasm/vfs/async_opfs/client.dart | 2 +- sqlite3/lib/src/wasm/vfs/indexed_db.dart | 90 ++++- sqlite3/lib/src/wasm/vfs/simple_opfs.dart | 2 +- sqlite3/lib/wasm.dart | 3 +- sqlite3/pubspec.yaml | 11 +- sqlite3/test/common/vfs.dart | 105 ++++++ sqlite3/test/ffi/vfs_test.dart | 11 + sqlite3/test/wasm/file_system_test.dart | 5 + sqlite3/test/wasm/utils.dart | 8 +- sqlite3_flutter_libs/CHANGELOG.md | 4 + sqlite3_flutter_libs/android/build.gradle | 2 +- .../ios/sqlite3_flutter_libs.podspec | 2 +- sqlite3_flutter_libs/linux/CMakeLists.txt | 4 +- .../macos/sqlite3_flutter_libs.podspec | 2 +- sqlite3_flutter_libs/pubspec.yaml | 2 +- sqlite3_flutter_libs/windows/CMakeLists.txt | 4 +- sqlite3_test/.gitignore | 7 + sqlite3_test/CHANGELOG.md | 3 + sqlite3_test/README.md | 76 +++++ sqlite3_test/analysis_options.yaml | 1 + .../example/sqlite3_test_example.dart | 29 ++ sqlite3_test/lib/sqlite3_test.dart | 188 +++++++++++ sqlite3_test/pubspec.yaml | 26 ++ sqlite3_test/test/sqlite3_test_test.dart | 65 ++++ sqlite3_web/CHANGELOG.md | 4 + sqlite3_web/lib/src/client.dart | 78 ++++- sqlite3_web/lib/src/database.dart | 20 +- sqlite3_web/lib/src/protocol.dart | 36 +- sqlite3_web/lib/src/types.dart | 14 +- sqlite3_web/lib/src/worker.dart | 91 ++++- sqlite3_web/pubspec.yaml | 4 +- sqlite3_web/test/integration_test.dart | 39 ++- sqlite3_web/tool/server.dart | 32 +- sqlite3_web/web/main.dart | 51 ++- 68 files changed, 1720 insertions(+), 280 deletions(-) rename sqlite3/lib/src/{wasm/vfs/memory.dart => in_memory_vfs.dart} (89%) rename sqlite3/lib/src/{wasm/vfs => }/utils.dart (100%) create mode 100644 sqlite3/test/common/vfs.dart create mode 100644 sqlite3/test/ffi/vfs_test.dart create mode 100644 sqlite3_test/.gitignore create mode 100644 sqlite3_test/CHANGELOG.md create mode 100644 sqlite3_test/README.md create mode 100644 sqlite3_test/analysis_options.yaml create mode 100644 sqlite3_test/example/sqlite3_test_example.dart create mode 100644 sqlite3_test/lib/sqlite3_test.dart create mode 100644 sqlite3_test/pubspec.yaml create mode 100644 sqlite3_test/test/sqlite3_test_test.dart diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 644e8e16..d1275368 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -173,7 +173,13 @@ jobs: dart-dependencies-${{ matrix.dart }}- dart-dependencies- - - name: Test + - name: Test sqlite3 package + run: | + dart pub get + dart test -P ci + working-directory: sqlite3/ + + - name: Test sqlite3_test package run: | dart pub get dart test -P ci @@ -249,8 +255,8 @@ jobs: - name: Start simulator run: | - IPHONE12=$(xcrun xctrace list devices 2>&1 | grep -m 1 "iPhone 14 Pro" | awk -F'[()]' '{print $4}') - xcrun simctl boot $IPHONE12 + IPHONE=$(xcrun xctrace list devices 2>&1 | grep -m 1 "iPhone" | awk -F'[()]' '{print $4}') + xcrun simctl boot $IPHONE - uses: actions/checkout@v4 - uses: subosito/flutter-action@v2.4.0 diff --git a/README.md b/README.md index aa4a94f0..79087032 100644 --- a/README.md +++ b/README.md @@ -3,12 +3,16 @@ This project contains Dart packages to use SQLite from Dart via `dart:ffi`. The main package in this repository is [`sqlite3`](sqlite3), which contains all the Dart apis and their implementation. -`package:sqlite3` is a pure-Dart package without a dependency on Flutter. +`package:sqlite3` is a pure-Dart package without a dependency on Flutter. It can be used both in Flutter apps or in standalone Dart applications. The `sqlite3_flutter_libs` and `sqlcipher_flutter_libs` packages contain no Dart code at all. Flutter users can depend on one of them to include native libraries in their apps. +`package:sqlite3_test` contains utilities that make integrating SQLite databases into Dart tests easier. +In particular, they patch `CURRENT_TIMESTAMP` and related constructs to return the (potentially faked) time +returned by `package:clock`. + ## Example Usage A file with basic usage examples for pure Dart can be found [here](sqlite3/example/main.dart). diff --git a/integration_tests/flutter_libs/.metadata b/integration_tests/flutter_libs/.metadata index 18cd0c71..2d1be89a 100644 --- a/integration_tests/flutter_libs/.metadata +++ b/integration_tests/flutter_libs/.metadata @@ -4,7 +4,7 @@ # This file should be version controlled and should not be manually edited. version: - revision: "2524052335ec76bb03e04ede244b071f1b86d190" + revision: "2663184aa79047d0a33a14a3b607954f8fdd8730" channel: "stable" project_type: app @@ -13,26 +13,26 @@ project_type: app migration: platforms: - platform: root - create_revision: 2524052335ec76bb03e04ede244b071f1b86d190 - base_revision: 2524052335ec76bb03e04ede244b071f1b86d190 + create_revision: 2663184aa79047d0a33a14a3b607954f8fdd8730 + base_revision: 2663184aa79047d0a33a14a3b607954f8fdd8730 - platform: android - create_revision: 2524052335ec76bb03e04ede244b071f1b86d190 - base_revision: 2524052335ec76bb03e04ede244b071f1b86d190 + create_revision: 2663184aa79047d0a33a14a3b607954f8fdd8730 + base_revision: 2663184aa79047d0a33a14a3b607954f8fdd8730 - platform: ios - create_revision: 2524052335ec76bb03e04ede244b071f1b86d190 - base_revision: 2524052335ec76bb03e04ede244b071f1b86d190 + create_revision: 2663184aa79047d0a33a14a3b607954f8fdd8730 + base_revision: 2663184aa79047d0a33a14a3b607954f8fdd8730 - platform: linux - create_revision: 2524052335ec76bb03e04ede244b071f1b86d190 - base_revision: 2524052335ec76bb03e04ede244b071f1b86d190 + create_revision: 2663184aa79047d0a33a14a3b607954f8fdd8730 + base_revision: 2663184aa79047d0a33a14a3b607954f8fdd8730 - platform: macos - create_revision: 2524052335ec76bb03e04ede244b071f1b86d190 - base_revision: 2524052335ec76bb03e04ede244b071f1b86d190 + create_revision: 2663184aa79047d0a33a14a3b607954f8fdd8730 + base_revision: 2663184aa79047d0a33a14a3b607954f8fdd8730 - platform: web - create_revision: 2524052335ec76bb03e04ede244b071f1b86d190 - base_revision: 2524052335ec76bb03e04ede244b071f1b86d190 + create_revision: 2663184aa79047d0a33a14a3b607954f8fdd8730 + base_revision: 2663184aa79047d0a33a14a3b607954f8fdd8730 - platform: windows - create_revision: 2524052335ec76bb03e04ede244b071f1b86d190 - base_revision: 2524052335ec76bb03e04ede244b071f1b86d190 + create_revision: 2663184aa79047d0a33a14a3b607954f8fdd8730 + base_revision: 2663184aa79047d0a33a14a3b607954f8fdd8730 # User provided section diff --git a/integration_tests/flutter_libs/android/.gitignore b/integration_tests/flutter_libs/android/.gitignore index 0a741cb4..55afd919 100644 --- a/integration_tests/flutter_libs/android/.gitignore +++ b/integration_tests/flutter_libs/android/.gitignore @@ -7,5 +7,7 @@ gradle-wrapper.jar GeneratedPluginRegistrant.java # Remember to never publicly share your keystore. -# See https://flutter.dev/docs/deployment/android#reference-the-keystore-from-the-app +# See https://flutter.dev/to/reference-keystore key.properties +**/*.keystore +**/*.jks diff --git a/integration_tests/flutter_libs/android/app/build.gradle b/integration_tests/flutter_libs/android/app/build.gradle index 57448529..7c9b9203 100644 --- a/integration_tests/flutter_libs/android/app/build.gradle +++ b/integration_tests/flutter_libs/android/app/build.gradle @@ -1,63 +1,43 @@ -def localProperties = new Properties() -def localPropertiesFile = rootProject.file('local.properties') -if (localPropertiesFile.exists()) { - localPropertiesFile.withReader('UTF-8') { reader -> - localProperties.load(reader) - } -} - -def flutterRoot = localProperties.getProperty('flutter.sdk') -if (flutterRoot == null) { - throw new GradleException("Flutter SDK not found. Define location with flutter.sdk in the local.properties file.") -} - -def flutterVersionCode = localProperties.getProperty('flutter.versionCode') -if (flutterVersionCode == null) { - flutterVersionCode = '1' +plugins { + id "com.android.application" + id "kotlin-android" + // The Flutter Gradle Plugin must be applied after the Android and Kotlin Gradle plugins. + id "dev.flutter.flutter-gradle-plugin" } -def flutterVersionName = localProperties.getProperty('flutter.versionName') -if (flutterVersionName == null) { - flutterVersionName = '1.0' -} - -apply plugin: 'com.android.application' -apply plugin: 'kotlin-android' -apply from: "$flutterRoot/packages/flutter_tools/gradle/flutter.gradle" - android { - compileSdkVersion 34 + namespace = "com.example.flutter_libs" + compileSdk = flutter.compileSdkVersion - sourceSets { - main.java.srcDirs += 'src/main/kotlin' + compileOptions { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 } - lintOptions { - disable 'InvalidPackage' + kotlinOptions { + jvmTarget = JavaVersion.VERSION_1_8 } defaultConfig { // TODO: Specify your own unique Application ID (https://developer.android.com/studio/build/application-id.html). - applicationId "com.example.flutter_libs" - minSdkVersion 21 - targetSdkVersion 28 - versionCode flutterVersionCode.toInteger() - versionName flutterVersionName + applicationId = "com.example.flutter_libs" + // You can update the following values to match your application needs. + // For more information, see: https://flutter.dev/to/review-gradle-config. + minSdk = flutter.minSdkVersion + targetSdk = flutter.targetSdkVersion + versionCode = flutter.versionCode + versionName = flutter.versionName } buildTypes { release { // TODO: Add your own signing config for the release build. // Signing with the debug keys for now, so `flutter run --release` works. - signingConfig signingConfigs.debug + signingConfig = signingConfigs.debug } } } flutter { - source '../..' -} - -dependencies { - implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" + source = "../.." } diff --git a/integration_tests/flutter_libs/android/app/src/debug/AndroidManifest.xml b/integration_tests/flutter_libs/android/app/src/debug/AndroidManifest.xml index 6749a11b..399f6981 100644 --- a/integration_tests/flutter_libs/android/app/src/debug/AndroidManifest.xml +++ b/integration_tests/flutter_libs/android/app/src/debug/AndroidManifest.xml @@ -1,6 +1,6 @@ - - diff --git a/integration_tests/flutter_libs/android/app/src/main/AndroidManifest.xml b/integration_tests/flutter_libs/android/app/src/main/AndroidManifest.xml index bc43d105..5ab95cf0 100644 --- a/integration_tests/flutter_libs/android/app/src/main/AndroidManifest.xml +++ b/integration_tests/flutter_libs/android/app/src/main/AndroidManifest.xml @@ -1,17 +1,13 @@ - - + - - @@ -44,4 +31,15 @@ android:name="flutterEmbedding" android:value="2" /> + + + + + + + diff --git a/integration_tests/flutter_libs/android/app/src/main/kotlin/com/example/flutter_libs/MainActivity.kt b/integration_tests/flutter_libs/android/app/src/main/kotlin/com/example/flutter_libs/MainActivity.kt index 8455fcfc..d27c0c9a 100644 --- a/integration_tests/flutter_libs/android/app/src/main/kotlin/com/example/flutter_libs/MainActivity.kt +++ b/integration_tests/flutter_libs/android/app/src/main/kotlin/com/example/flutter_libs/MainActivity.kt @@ -2,5 +2,4 @@ package com.example.flutter_libs import io.flutter.embedding.android.FlutterActivity -class MainActivity: FlutterActivity() { -} +class MainActivity: FlutterActivity() diff --git a/integration_tests/flutter_libs/android/app/src/main/res/values/styles.xml b/integration_tests/flutter_libs/android/app/src/main/res/values/styles.xml index 1f83a33f..cb1ef880 100644 --- a/integration_tests/flutter_libs/android/app/src/main/res/values/styles.xml +++ b/integration_tests/flutter_libs/android/app/src/main/res/values/styles.xml @@ -1,18 +1,18 @@ - - - diff --git a/integration_tests/flutter_libs/android/app/src/profile/AndroidManifest.xml b/integration_tests/flutter_libs/android/app/src/profile/AndroidManifest.xml index 6749a11b..399f6981 100644 --- a/integration_tests/flutter_libs/android/app/src/profile/AndroidManifest.xml +++ b/integration_tests/flutter_libs/android/app/src/profile/AndroidManifest.xml @@ -1,6 +1,6 @@ - - diff --git a/integration_tests/flutter_libs/android/build.gradle b/integration_tests/flutter_libs/android/build.gradle index c732d03f..d2ffbffa 100644 --- a/integration_tests/flutter_libs/android/build.gradle +++ b/integration_tests/flutter_libs/android/build.gradle @@ -1,29 +1,16 @@ -buildscript { - ext.kotlin_version = '1.6.10' - repositories { - google() - jcenter() - } - - dependencies { - classpath 'com.android.tools.build:gradle:7.0.0' - classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" - } -} - allprojects { repositories { google() - jcenter() + mavenCentral() } } -rootProject.buildDir = '../build' +rootProject.buildDir = "../build" subprojects { project.buildDir = "${rootProject.buildDir}/${project.name}" } subprojects { - project.evaluationDependsOn(':app') + project.evaluationDependsOn(":app") } tasks.register("clean", Delete) { diff --git a/integration_tests/flutter_libs/android/gradle.properties b/integration_tests/flutter_libs/android/gradle.properties index 38c8d454..25971708 100644 --- a/integration_tests/flutter_libs/android/gradle.properties +++ b/integration_tests/flutter_libs/android/gradle.properties @@ -1,4 +1,3 @@ -org.gradle.jvmargs=-Xmx1536M -android.enableR8=true +org.gradle.jvmargs=-Xmx4G -XX:MaxMetaspaceSize=2G -XX:+HeapDumpOnOutOfMemoryError android.useAndroidX=true android.enableJetifier=true diff --git a/integration_tests/flutter_libs/android/gradle/wrapper/gradle-wrapper.properties b/integration_tests/flutter_libs/android/gradle/wrapper/gradle-wrapper.properties index 02e5f581..afa1e8eb 100644 --- a/integration_tests/flutter_libs/android/gradle/wrapper/gradle-wrapper.properties +++ b/integration_tests/flutter_libs/android/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,5 @@ -#Fri Jun 23 08:50:38 CEST 2017 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-all.zip diff --git a/integration_tests/flutter_libs/android/settings.gradle b/integration_tests/flutter_libs/android/settings.gradle index d3b6a401..e2be9b48 100644 --- a/integration_tests/flutter_libs/android/settings.gradle +++ b/integration_tests/flutter_libs/android/settings.gradle @@ -1,15 +1,25 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. +pluginManagement { + def flutterSdkPath = { + def properties = new Properties() + file("local.properties").withInputStream { properties.load(it) } + def flutterSdkPath = properties.getProperty("flutter.sdk") + assert flutterSdkPath != null, "flutter.sdk not set in local.properties" + return flutterSdkPath + }() -include ':app' + includeBuild("$flutterSdkPath/packages/flutter_tools/gradle") -def localPropertiesFile = new File(rootProject.projectDir, "local.properties") -def properties = new Properties() + repositories { + google() + mavenCentral() + gradlePluginPortal() + } +} -assert localPropertiesFile.exists() -localPropertiesFile.withReader("UTF-8") { reader -> properties.load(reader) } +plugins { + id "dev.flutter.flutter-plugin-loader" version "1.0.0" + id "com.android.application" version "8.7.0" apply false + id "org.jetbrains.kotlin.android" version "1.8.22" apply false +} -def flutterSdkPath = properties.getProperty("flutter.sdk") -assert flutterSdkPath != null, "flutter.sdk not set in local.properties" -apply from: "$flutterSdkPath/packages/flutter_tools/gradle/app_plugin_loader.gradle" +include ":app" diff --git a/integration_tests/flutter_libs/pubspec.yaml b/integration_tests/flutter_libs/pubspec.yaml index 719a548f..c1397d33 100644 --- a/integration_tests/flutter_libs/pubspec.yaml +++ b/integration_tests/flutter_libs/pubspec.yaml @@ -5,7 +5,7 @@ publish_to: 'none' version: 1.0.0+1 environment: - sdk: ">=2.12.0-0 <3.0.0" + sdk: ^3.5.0 dependencies: sqlite3_flutter_libs: diff --git a/sqlite3/CHANGELOG.md b/sqlite3/CHANGELOG.md index c1bde2c2..674f791a 100644 --- a/sqlite3/CHANGELOG.md +++ b/sqlite3/CHANGELOG.md @@ -1,3 +1,10 @@ +## 2.5.0 + +- Allow registering custom virtual file systems on all platforms. Previously, + this was only supported on the web. +- IndexedDB file system: Store `ArrayBuffer`s instead of `Blob`s when it looks + like storing blobs would cause issues. + ## 2.4.7 - Web: Improve performance of in-memory and IndexedDB file system implementations. @@ -64,11 +71,11 @@ ## 2.0.0 -- __Breaking__: The WASM implementation no longer registers a default virtual +- **Breaking**: The WASM implementation no longer registers a default virtual file system. Instead, `registerVirtualFileSystem` needs to be used to add desired file system implementations. -- __Breaking__: Fix a typo, `CommmonSqlite3` is now called `CommonSqlite3`. -- __Breaking__: Introduce class modifiers on classes of this package that aren't +- **Breaking**: Fix a typo, `CommmonSqlite3` is now called `CommonSqlite3`. +- **Breaking**: Introduce class modifiers on classes of this package that aren't meant to be extended or implemented by users. - Add `PreparedStatement.reset()`. - Add the `CustomStatementParameter` class which can be passed as a statement @@ -149,7 +156,7 @@ - Add support for application-defined window functions. To register a custom window function, implement `WindowFunction` and register your function with `database.registerAggregateFunction`. -- __Breaking__ (For the experimental `package:sqlite3/wasm.dart` library): +- **Breaking** (For the experimental `package:sqlite3/wasm.dart` library): - The IndexedDB implementation now stores data in 4k blocks instead of full files. - Removed `IndexedDbFileSystem.load`. Use `IndexedDbFileSystem.open` instead. - An `IndexedDbFileSystem` now stores all files, the concept of a persistence diff --git a/sqlite3/assets/sqlite3.h b/sqlite3/assets/sqlite3.h index a6bd5c00..e457ca22 100644 --- a/sqlite3/assets/sqlite3.h +++ b/sqlite3/assets/sqlite3.h @@ -130,3 +130,81 @@ int sqlite3_auto_extension(void *xEntryPoint); // Database configuration int sqlite3_db_config(sqlite3 *db, int op, ...); + +// VFS +typedef struct sqlite3_file sqlite3_file; + +struct sqlite3_io_methods { + int iVersion; + int (*xClose)(sqlite3_file *); + int (*xRead)(sqlite3_file *, void *, int iAmt, int64_t iOfst); + int (*xWrite)(sqlite3_file *, const void *, int iAmt, int64_t iOfst); + int (*xTruncate)(sqlite3_file *, int64_t size); + int (*xSync)(sqlite3_file *, int flags); + int (*xFileSize)(sqlite3_file *, int64_t *pSize); + int (*xLock)(sqlite3_file *, int); + int (*xUnlock)(sqlite3_file *, int); + int (*xCheckReservedLock)(sqlite3_file *, int *pResOut); + int (*xFileControl)(sqlite3_file *, int op, void *pArg); + int (*xSectorSize)(sqlite3_file *); + int (*xDeviceCharacteristics)(sqlite3_file *); + /* Methods above are valid for version 1 */ + int (*xShmMap)(sqlite3_file *, int iPg, int pgsz, int, void **); + int (*xShmLock)(sqlite3_file *, int offset, int n, int flags); + void (*xShmBarrier)(sqlite3_file *); + int (*xShmUnmap)(sqlite3_file *, int deleteFlag); + /* Methods above are valid for version 2 */ + int (*xFetch)(sqlite3_file *, int64_t iOfst, int iAmt, void **pp); + int (*xUnfetch)(sqlite3_file *, int64_t iOfst, void *p); + /* Methods above are valid for version 3 */ + /* Additional methods may be added in future releases */ +}; + +struct sqlite3_file { + const struct sqlite3_io_methods *pMethods; /* Methods for an open file */ +}; + +typedef struct sqlite3_vfs sqlite3_vfs; +typedef void (*sqlite3_syscall_ptr)(void); +typedef const char *sqlite3_filename; + +struct sqlite3_vfs { + int iVersion; /* Structure version number (currently 3) */ + int szOsFile; /* Size of subclassed sqlite3_file */ + int mxPathname; /* Maximum file pathname length */ + sqlite3_vfs *pNext; /* Next registered VFS */ + const char *zName; /* Name of this virtual file system */ + void *pAppData; /* Pointer to application-specific data */ + int (*xOpen)(sqlite3_vfs *, sqlite3_filename zName, sqlite3_file *, int flags, + int *pOutFlags); + int (*xDelete)(sqlite3_vfs *, const char *zName, int syncDir); + int (*xAccess)(sqlite3_vfs *, const char *zName, int flags, int *pResOut); + int (*xFullPathname)(sqlite3_vfs *, const char *zName, int nOut, char *zOut); + void *(*xDlOpen)(sqlite3_vfs *, const char *zFilename); + void (*xDlError)(sqlite3_vfs *, int nByte, char *zErrMsg); + void (*(*xDlSym)(sqlite3_vfs *, void *, const char *zSymbol))(void); + void (*xDlClose)(sqlite3_vfs *, void *); + int (*xRandomness)(sqlite3_vfs *, int nByte, char *zOut); + int (*xSleep)(sqlite3_vfs *, int microseconds); + int (*xCurrentTime)(sqlite3_vfs *, double *); + int (*xGetLastError)(sqlite3_vfs *, int, char *); + /* + ** The methods above are in version 1 of the sqlite_vfs object + ** definition. Those that follow are added in version 2 or later + */ + int (*xCurrentTimeInt64)(sqlite3_vfs *, int64_t *); + /* + ** The methods above are in versions 1 and 2 of the sqlite_vfs object. + ** Those below are for version 3 and greater. + */ + int (*xSetSystemCall)(sqlite3_vfs *, const char *zName, sqlite3_syscall_ptr); + sqlite3_syscall_ptr (*xGetSystemCall)(sqlite3_vfs *, const char *zName); + const char *(*xNextSystemCall)(sqlite3_vfs *, const char *zName); + /* + ** The methods above are in versions 1 through 3 of the sqlite_vfs object. + ** New fields may be appended in future versions. The iVersion + ** value will increment whenever this happens. + */ +}; +int sqlite3_vfs_register(sqlite3_vfs *, int makeDflt); +int sqlite3_vfs_unregister(sqlite3_vfs *); diff --git a/sqlite3/assets/wasm/CMakeLists.txt b/sqlite3/assets/wasm/CMakeLists.txt index 1619201c..58975fba 100644 --- a/sqlite3/assets/wasm/CMakeLists.txt +++ b/sqlite3/assets/wasm/CMakeLists.txt @@ -11,7 +11,8 @@ include(FetchContent) FetchContent_Declare( sqlite3 - URL https://sqlite.org/2024/sqlite-autoconf-3460000.tar.gz + # NOTE: When changing this, also update `test/wasm/sqlite3_test.dart` + URL https://sqlite.org/2024/sqlite-autoconf-3470000.tar.gz DOWNLOAD_EXTRACT_TIMESTAMP NEW ) diff --git a/sqlite3/lib/common.dart b/sqlite3/lib/common.dart index e04c296a..f081d32f 100644 --- a/sqlite3/lib/common.dart +++ b/sqlite3/lib/common.dart @@ -1,6 +1,6 @@ /// Exports common interfaces that are implemented by both the `dart:ffi` and /// the `dart:js` WASM version of this library. -library sqlite3.common; +library; export 'src/constants.dart'; export 'src/database.dart'; @@ -11,3 +11,4 @@ export 'src/sqlite3.dart'; export 'src/statement.dart' show CommonPreparedStatement, StatementParameters, CustomStatementParameter; export 'src/vfs.dart'; +export 'src/in_memory_vfs.dart' show InMemoryFileSystem; diff --git a/sqlite3/lib/open.dart b/sqlite3/lib/open.dart index dc2776bf..5362e552 100644 --- a/sqlite3/lib/open.dart +++ b/sqlite3/lib/open.dart @@ -1,6 +1,6 @@ /// Utils to open a [DynamicLibrary] on platforms that aren't supported by /// default. -library open; +library; import 'dart:ffi'; diff --git a/sqlite3/lib/sqlite3.dart b/sqlite3/lib/sqlite3.dart index 5df22e8d..b63aa9b8 100644 --- a/sqlite3/lib/sqlite3.dart +++ b/sqlite3/lib/sqlite3.dart @@ -1,5 +1,5 @@ /// Dart bindings to `sqlite3`. -library sqlite3; +library; // Hide common base classes that have more specific ffi-APIs. export 'common.dart' hide CommonPreparedStatement, CommonDatabase; diff --git a/sqlite3/lib/src/ffi/api.dart b/sqlite3/lib/src/ffi/api.dart index 829941cb..06b34fe3 100644 --- a/sqlite3/lib/src/ffi/api.dart +++ b/sqlite3/lib/src/ffi/api.dart @@ -31,7 +31,7 @@ abstract interface class Sqlite3 implements CommonSqlite3 { Database fromPointer(Pointer database); @override - Database openInMemory(); + Database openInMemory({String? vfs}); /// Opens a new in-memory database and copies another database into it /// https://www.sqlite.org/c3ref/backup_finish.html diff --git a/sqlite3/lib/src/ffi/bindings.dart b/sqlite3/lib/src/ffi/bindings.dart index 3589968f..c58ec444 100644 --- a/sqlite3/lib/src/ffi/bindings.dart +++ b/sqlite3/lib/src/ffi/bindings.dart @@ -3,9 +3,12 @@ import 'dart:convert'; import 'dart:ffi'; import 'dart:typed_data'; +import 'package:ffi/ffi.dart' as ffi; import 'package:meta/meta.dart'; +import 'package:sqlite3/src/vfs.dart'; import '../constants.dart'; +import '../exception.dart'; import '../functions.dart'; import '../implementation/bindings.dart'; import 'memory.dart'; @@ -62,6 +65,7 @@ class BindingsWithLibrary { final class FfiBindings extends RawSqliteBindings { final BindingsWithLibrary bindings; + final _vfsPointers = Expando<_RegisteredVfs>(); FfiBindings(this.bindings); @@ -119,6 +123,286 @@ final class FfiBindings extends RawSqliteBindings { String sqlite3_sourceid() { return bindings.bindings.sqlite3_sourceid().readString(); } + + @override + void registerVirtualFileSystem(VirtualFileSystem vfs, int makeDefault) { + final ptr = _RegisteredVfs.allocate(vfs); + final result = + bindings.bindings.sqlite3_vfs_register(ptr._vfsPtr, makeDefault); + if (result != SqlError.SQLITE_OK) { + ptr.deallocate(); + throw SqliteException(result, 'Could not register VFS.'); + } + + _vfsPointers[vfs] = ptr; + } + + @override + void unregisterVirtualFileSystem(VirtualFileSystem vfs) { + final ptr = _vfsPointers[vfs]; + if (ptr == null) { + throw StateError('vfs has not been registered'); + } + + final result = bindings.bindings.sqlite3_vfs_unregister(ptr._vfsPtr); + if (result != SqlError.SQLITE_OK) { + throw SqliteException(result, 'Could not unregister VFS.'); + } + + ptr.deallocate(); + } +} + +final class _RegisteredVfs { + static final Map _files = {}; + static final Map _vfs = {}; + + static int _vfsCounter = 0; + static int _fileCounter = 0; + + final Pointer _vfsPtr; + final Pointer _name; + + _RegisteredVfs(this._vfsPtr, this._name); + + factory _RegisteredVfs.allocate(VirtualFileSystem dartVfs) { + final name = Utf8Utils.allocateZeroTerminated(dartVfs.name).cast(); + final id = _vfsCounter++; + + final vfs = ffi.calloc(); + vfs.ref + ..iVersion = 2 // We don't support syscalls yet + ..szOsFile = sizeOf<_DartFile>() + ..mxPathname = 1024 + ..zName = name + ..pAppData = Pointer.fromAddress(id) + ..xOpen = Pointer.fromFunction(_xOpen, SqlError.SQLITE_ERROR) + ..xDelete = Pointer.fromFunction(_xDelete, SqlError.SQLITE_ERROR) + ..xAccess = Pointer.fromFunction(_xAccess, SqlError.SQLITE_ERROR) + ..xFullPathname = + Pointer.fromFunction(_xFullPathname, SqlError.SQLITE_ERROR) + ..xDlOpen = nullPtr() + ..xDlError = nullPtr() + ..xDlSym = nullPtr() + ..xDlClose = nullPtr() + ..xRandomness = Pointer.fromFunction(_xRandomness, SqlError.SQLITE_ERROR) + ..xSleep = Pointer.fromFunction(_xSleep, SqlError.SQLITE_ERROR) + ..xCurrentTime = nullPtr() + ..xGetLastError = nullPtr() + ..xCurrentTimeInt64 = + Pointer.fromFunction(_xCurrentTime64, SqlError.SQLITE_ERROR); + + _vfs[id] = dartVfs; + return _RegisteredVfs(vfs, name); + } + + void deallocate() { + _vfs.remove(_vfsPtr.ref.pAppData.address); + ffi.calloc.free(_vfsPtr); + _name.free(); + } + + static int _runVfs( + Pointer vfs, void Function(VirtualFileSystem) body) { + final dartVfs = _vfs[vfs.ref.pAppData.address]!; + try { + body(dartVfs); + return SqlError.SQLITE_OK; + } on VfsException catch (e) { + return e.returnCode; + } on Object { + return SqlError.SQLITE_ERROR; + } + } + + static int _xOpen(Pointer vfsPtr, Pointer zName, + Pointer file, int flags, Pointer pOutFlags) { + return _runVfs(vfsPtr, (vfs) { + final fileName = Sqlite3Filename(zName.cast().readString()); + final dartFilePtr = file.cast<_DartFile>(); + + final (file: dartFile, :outFlags) = vfs.xOpen(fileName, flags); + final fileId = _fileCounter++; + _files[fileId] = dartFile; + + final ioMethods = ffi.calloc(); + ioMethods.ref + ..iVersion = 1 + ..xClose = Pointer.fromFunction(_xClose, SqlError.SQLITE_ERROR) + ..xRead = Pointer.fromFunction(_xRead, SqlError.SQLITE_ERROR) + ..xWrite = Pointer.fromFunction(_xWrite, SqlError.SQLITE_ERROR) + ..xTruncate = Pointer.fromFunction(_xTruncate, SqlError.SQLITE_ERROR) + ..xSync = Pointer.fromFunction(_xSync, SqlError.SQLITE_ERROR) + ..xFileSize = Pointer.fromFunction(_xFileSize, SqlError.SQLITE_ERROR) + ..xLock = Pointer.fromFunction(_xLock, SqlError.SQLITE_ERROR) + ..xUnlock = Pointer.fromFunction(_xUnlock, SqlError.SQLITE_ERROR) + ..xCheckReservedLock = + Pointer.fromFunction(_xCheckReservedLock, SqlError.SQLITE_ERROR) + ..xFileControl = + Pointer.fromFunction(_xFileControl, SqlError.SQLITE_NOTFOUND) + ..xSectorSize = Pointer.fromFunction(_xSectorSize, 4096) + ..xDeviceCharacteristics = + Pointer.fromFunction(_xDeviveCharacteristics, 0); + + if (!pOutFlags.isNullPointer) { + pOutFlags.value = outFlags; + } + + dartFilePtr.ref + ..pMethods = ioMethods + ..dartFileId = fileId; + }); + } + + static int _xDelete( + Pointer vfsPtr, Pointer zName, int syncDir) { + return _runVfs(vfsPtr, + (vfs) => vfs.xDelete(zName.cast().readString(), syncDir)); + } + + static int _xAccess(Pointer vfsPtr, Pointer zName, + int flags, Pointer pResOut) { + return _runVfs(vfsPtr, (vfs) { + if (!pResOut.isNullPointer) { + pResOut.value = + vfs.xAccess(zName.cast().readString(), flags); + } + }); + } + + static int _xFullPathname(Pointer vfsPtr, Pointer zName, + int nOut, Pointer zOut) { + return _runVfs(vfsPtr, (vfs) { + final bytes = utf8 + .encode(vfs.xFullPathName(zName.cast().readString())); + if (bytes.length >= nOut) { + throw VfsException(SqlError.SQLITE_TOOBIG); + } + + final target = zOut.cast().asTypedList(nOut); + target.setAll(0, bytes); + target[bytes.length] = 0; + }); + } + + static int _xRandomness( + Pointer vfsPtr, int nByte, Pointer zOut) { + return _runVfs(vfsPtr, (vfs) { + vfs.xRandomness(zOut.cast().asTypedList(nByte)); + }); + } + + static int _xSleep(Pointer vfsPtr, int microseconds) { + return _runVfs( + vfsPtr, (vfs) => vfs.xSleep(Duration(microseconds: microseconds))); + } + + static int _xCurrentTime64(Pointer vfsPtr, Pointer out) { + return _runVfs(vfsPtr, (vfs) { + if (!out.isNullPointer) { + // https://github.com/sqlite/sqlite/blob/8ee75f7c3ac1456b8d941781857be27bfddb57d6/src/os_unix.c#L6757 + const unixEpoch = 24405875 * 8640000; + + out.value = unixEpoch + vfs.xCurrentTime().millisecondsSinceEpoch; + } + }); + } + + static int _runFile( + Pointer file, void Function(VirtualFileSystemFile) body) { + final id = file.cast<_DartFile>().ref.dartFileId; + final dartFile = _files[id]!; + try { + body(dartFile); + return SqlError.SQLITE_OK; + } on VfsException catch (e) { + return e.returnCode; + } on Object { + return SqlError.SQLITE_ERROR; + } + } + + static int _xClose(Pointer ptr) { + return _runFile(ptr, (file) { + file.xClose(); + + final dartFile = ptr.cast<_DartFile>().ref; + _files.remove(dartFile.dartFileId); + ffi.calloc.free(dartFile.pMethods); + }); + } + + static int _xRead( + Pointer ptr, Pointer target, int amount, int offset) { + return _runFile(ptr, (file) { + final buffer = target.cast().asTypedList(amount); + file.xRead(buffer, offset); + }); + } + + static int _xWrite( + Pointer ptr, Pointer target, int amount, int offset) { + return _runFile(ptr, (file) { + final buffer = target.cast().asTypedList(amount); + file.xWrite(buffer, offset); + }); + } + + static int _xTruncate(Pointer ptr, int size) { + return _runFile(ptr, (file) => file.xTruncate(size)); + } + + static int _xSync(Pointer ptr, int flags) { + return _runFile(ptr, (file) => file.xSync(flags)); + } + + static int _xFileSize(Pointer ptr, Pointer pSize) { + return _runFile(ptr, (file) { + if (!pSize.isNullPointer) { + pSize.value = file.xFileSize(); + } + }); + } + + static int _xLock(Pointer ptr, int flags) { + return _runFile(ptr, (file) => file.xLock(flags)); + } + + static int _xUnlock(Pointer ptr, int flags) { + return _runFile(ptr, (file) => file.xUnlock(flags)); + } + + static int _xCheckReservedLock( + Pointer ptr, Pointer pResOut) { + return _runFile(ptr, (file) { + if (!pResOut.isNullPointer) { + pResOut.value = file.xCheckReservedLock(); + } + }); + } + + static int _xFileControl( + Pointer ptr, int op, Pointer pArg) { + // We don't currently support filecontrol operations in the VFS. + return SqlError.SQLITE_NOTFOUND; + } + + static int _xSectorSize(Pointer ptr) { + // We don't currently support custom sector sizes. + return 4096; + } + + static int _xDeviveCharacteristics(Pointer ptr) { + return _runFile(ptr, (file) => file.xDeviceCharacteristics); + } +} + +final class _DartFile extends Struct { + // extends sqlite3_file: + external Pointer pMethods; + // additional definitions + @Int64() + external int dartFileId; } final class FfiDatabase extends RawSqliteDatabase { @@ -266,6 +550,7 @@ final class FfiDatabase extends RawSqliteDatabase { final previous = _installedUpdateHook; if (hook == null) { + _installedUpdateHook = null; bindings.bindings.sqlite3_update_hook(db, nullPtr(), nullPtr()); } else { final native = _installedUpdateHook = hook.toNative(); @@ -299,13 +584,10 @@ final class FfiDatabase extends RawSqliteDatabase { static Pointer)>> _xDestroy( List callables) { - int destroy(Pointer _) { + void destroy(Pointer _) { for (final callable in callables) { callable.close(); } - - // TODO: Remove and change to void after Dart 3.5 or https://github.com/dart-lang/sdk/issues/56064 - return 0; } final callable = @@ -665,26 +947,6 @@ final class FfiContext extends RawSqliteContext { } } -class RegisteredFunctionSet { - final RawXFunc? xFunc; - final RawXStep? xStep; - final RawXFinal? xFinal; - - final RawXFinal? xValue; - final RawXStep? xInverse; - - final RawCollation? collation; - - RegisteredFunctionSet({ - this.xFunc, - this.xStep, - this.xFinal, - this.xValue, - this.xInverse, - this.collation, - }); -} - class _ValueList extends ListBase { @override int length; @@ -715,8 +977,6 @@ extension on RawXFunc { return NativeCallable.isolateLocal((Pointer ctx, int nArgs, Pointer> args) { this(FfiContext(bindings, ctx), _ValueList(nArgs, args, bindings)); - // TODO: Remove and change to void after Dart 3.5 or https://github.com/dart-lang/sdk/issues/56064 - return 0; }) ..keepIsolateAlive = false; } @@ -728,8 +988,6 @@ extension on RawXFinal { final context = FfiContext(bindings, ctx); this(context); if (clean) context.freeContext(); - // TODO: Remove and change to void after Dart 3.5 or https://github.com/dart-lang/sdk/issues/56064 - return 0; }) ..keepIsolateAlive = false; } @@ -762,9 +1020,6 @@ extension on RawUpdateHook { Pointer table, int rowid) { final tableName = table.readString(); this(kind, tableName, rowid); - - // TODO: Remove and change to void after Dart 3.5 or https://github.com/dart-lang/sdk/issues/56064 - return 0; }, )..keepIsolateAlive = false; } diff --git a/sqlite3/lib/src/ffi/implementation.dart b/sqlite3/lib/src/ffi/implementation.dart index 1a24ddc4..750ddc56 100644 --- a/sqlite3/lib/src/ffi/implementation.dart +++ b/sqlite3/lib/src/ffi/implementation.dart @@ -35,8 +35,8 @@ final class FfiSqlite3 extends Sqlite3Implementation implements Sqlite3 { } @override - FfiDatabaseImplementation openInMemory() { - return super.openInMemory() as FfiDatabaseImplementation; + FfiDatabaseImplementation openInMemory({String? vfs}) { + return super.openInMemory(vfs: vfs) as FfiDatabaseImplementation; } @override diff --git a/sqlite3/lib/src/ffi/memory.dart b/sqlite3/lib/src/ffi/memory.dart index 7d7850f7..6d356834 100644 --- a/sqlite3/lib/src/ffi/memory.dart +++ b/sqlite3/lib/src/ffi/memory.dart @@ -56,8 +56,9 @@ extension Utf8Utils on Pointer { String readString([int? length]) { final resolvedLength = length ??= _length; + final dartList = cast().asTypedList(resolvedLength); - return utf8.decode(cast().asTypedList(resolvedLength)); + return utf8.decode(dartList); } static Pointer allocateZeroTerminated(String string) { diff --git a/sqlite3/lib/src/ffi/sqlite3.g.dart b/sqlite3/lib/src/ffi/sqlite3.g.dart index 1109177f..288f485e 100644 --- a/sqlite3/lib/src/ffi/sqlite3.g.dart +++ b/sqlite3/lib/src/ffi/sqlite3.g.dart @@ -1358,6 +1358,37 @@ class Bindings { )>)>>('sqlite3_db_config'); late final _sqlite3_db_config = _sqlite3_db_configPtr.asFunction< int Function(ffi.Pointer, int, int, ffi.Pointer)>(); + + int sqlite3_vfs_register( + ffi.Pointer arg0, + int makeDflt, + ) { + return _sqlite3_vfs_register( + arg0, + makeDflt, + ); + } + + late final _sqlite3_vfs_registerPtr = _lookup< + ffi + .NativeFunction, ffi.Int)>>( + 'sqlite3_vfs_register'); + late final _sqlite3_vfs_register = _sqlite3_vfs_registerPtr + .asFunction, int)>(); + + int sqlite3_vfs_unregister( + ffi.Pointer arg0, + ) { + return _sqlite3_vfs_unregister( + arg0, + ); + } + + late final _sqlite3_vfs_unregisterPtr = + _lookup)>>( + 'sqlite3_vfs_unregister'); + late final _sqlite3_vfs_unregister = _sqlite3_vfs_unregisterPtr + .asFunction)>(); } final class sqlite3_char extends ffi.Opaque {} @@ -1373,3 +1404,206 @@ final class sqlite3_api_routines extends ffi.Opaque {} final class sqlite3_value extends ffi.Opaque {} final class sqlite3_context extends ffi.Opaque {} + +final class sqlite3_io_methods extends ffi.Struct { + @ffi.Int() + external int iVersion; + + external ffi + .Pointer)>> + xClose; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer, ffi.Pointer, + ffi.Int, ffi.Int64)>> xRead; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer, ffi.Pointer, + ffi.Int, ffi.Int64)>> xWrite; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer, ffi.Int64)>> xTruncate; + + external ffi.Pointer< + ffi + .NativeFunction, ffi.Int)>> + xSync; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function( + ffi.Pointer, ffi.Pointer)>> xFileSize; + + external ffi.Pointer< + ffi + .NativeFunction, ffi.Int)>> + xLock; + + external ffi.Pointer< + ffi + .NativeFunction, ffi.Int)>> + xUnlock; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function( + ffi.Pointer, ffi.Pointer)>> + xCheckReservedLock; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function( + ffi.Pointer, ffi.Int, ffi.Pointer)>> + xFileControl; + + external ffi + .Pointer)>> + xSectorSize; + + external ffi + .Pointer)>> + xDeviceCharacteristics; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer, ffi.Int, ffi.Int, ffi.Int, + ffi.Pointer>)>> xShmMap; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function( + ffi.Pointer, ffi.Int, ffi.Int, ffi.Int)>> xShmLock; + + external ffi + .Pointer)>> + xShmBarrier; + + external ffi.Pointer< + ffi + .NativeFunction, ffi.Int)>> + xShmUnmap; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer, ffi.Int64, ffi.Int, + ffi.Pointer>)>> xFetch; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function( + ffi.Pointer, ffi.Int64, ffi.Pointer)>> + xUnfetch; +} + +final class sqlite3_file extends ffi.Struct { + external ffi.Pointer pMethods; +} + +final class sqlite3_vfs extends ffi.Struct { + @ffi.Int() + external int iVersion; + + @ffi.Int() + external int szOsFile; + + @ffi.Int() + external int mxPathname; + + external ffi.Pointer pNext; + + external ffi.Pointer zName; + + external ffi.Pointer pAppData; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer, ffi.Pointer, + ffi.Pointer, ffi.Int, ffi.Pointer)>> xOpen; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function( + ffi.Pointer, ffi.Pointer, ffi.Int)>> + xDelete; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer, ffi.Pointer, + ffi.Int, ffi.Pointer)>> xAccess; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer, ffi.Pointer, + ffi.Int, ffi.Pointer)>> xFullPathname; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>> xDlOpen; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Void Function( + ffi.Pointer, ffi.Int, ffi.Pointer)>> + xDlError; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Pointer> Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>> xDlSym; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Void Function( + ffi.Pointer, ffi.Pointer)>> xDlClose; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function( + ffi.Pointer, ffi.Int, ffi.Pointer)>> + xRandomness; + + external ffi.Pointer< + ffi + .NativeFunction, ffi.Int)>> + xSleep; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function( + ffi.Pointer, ffi.Pointer)>> xCurrentTime; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function( + ffi.Pointer, ffi.Int, ffi.Pointer)>> + xGetLastError; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function( + ffi.Pointer, ffi.Pointer)>> + xCurrentTimeInt64; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer, ffi.Pointer, + ffi.Pointer>)>> + xSetSystemCall; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Pointer> Function( + ffi.Pointer, ffi.Pointer)>> xGetSystemCall; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>> + xNextSystemCall; +} diff --git a/sqlite3/lib/src/implementation/bindings.dart b/sqlite3/lib/src/implementation/bindings.dart index d7aaeb4f..02cf6be3 100644 --- a/sqlite3/lib/src/implementation/bindings.dart +++ b/sqlite3/lib/src/implementation/bindings.dart @@ -1,11 +1,12 @@ @internal -library sqlite3.implementation.bindings; +library; import 'dart:typed_data'; import 'package:meta/meta.dart'; import '../functions.dart'; +import '../vfs.dart'; // ignore_for_file: non_constant_identifier_names @@ -36,6 +37,9 @@ abstract base class RawSqliteBindings { String name, int flags, String? zVfs); String sqlite3_errstr(int extendedErrorCode); + + void registerVirtualFileSystem(VirtualFileSystem vfs, int makeDefault); + void unregisterVirtualFileSystem(VirtualFileSystem vfs); } /// Combines a sqlite result code and the result object. diff --git a/sqlite3/lib/src/implementation/sqlite3.dart b/sqlite3/lib/src/implementation/sqlite3.dart index 32b62a7d..34c1b7ac 100644 --- a/sqlite3/lib/src/implementation/sqlite3.dart +++ b/sqlite3/lib/src/implementation/sqlite3.dart @@ -3,6 +3,7 @@ import 'package:meta/meta.dart'; import '../constants.dart'; import '../database.dart'; import '../sqlite3.dart'; +import '../vfs.dart'; import 'bindings.dart'; import 'database.dart'; import 'exception.dart'; @@ -66,8 +67,19 @@ base class Sqlite3Implementation implements CommonSqlite3 { } @override - CommonDatabase openInMemory() { - return open(':memory:'); + CommonDatabase openInMemory({String? vfs}) { + return open(':memory:', vfs: vfs); + } + + @override + void registerVirtualFileSystem(VirtualFileSystem vfs, + {bool makeDefault = false}) { + bindings.registerVirtualFileSystem(vfs, makeDefault ? 1 : 0); + } + + @override + void unregisterVirtualFileSystem(VirtualFileSystem vfs) { + bindings.unregisterVirtualFileSystem(vfs); } @override diff --git a/sqlite3/lib/src/wasm/vfs/memory.dart b/sqlite3/lib/src/in_memory_vfs.dart similarity index 89% rename from sqlite3/lib/src/wasm/vfs/memory.dart rename to sqlite3/lib/src/in_memory_vfs.dart index 5d0cc7a4..ad1a5f24 100644 --- a/sqlite3/lib/src/wasm/vfs/memory.dart +++ b/sqlite3/lib/src/in_memory_vfs.dart @@ -4,10 +4,15 @@ import 'dart:typed_data'; import 'package:path/path.dart' as p; import 'package:typed_data/typed_buffers.dart'; -import '../../constants.dart'; -import '../../vfs.dart'; +import 'constants.dart'; +import 'vfs.dart'; import 'utils.dart'; +/// A virtual file system implementation that stores all files in memory. +/// +/// This file system is commonly used on the web as a buffer in front of +/// asynchronous storage APIs like IndexedDb. It can also serve as an example on +/// how to write custom file systems to be used with sqlite3. final class InMemoryFileSystem extends BaseVirtualFileSystem { final Map fileData = {}; diff --git a/sqlite3/lib/src/sqlite3.dart b/sqlite3/lib/src/sqlite3.dart index c7f5445b..05a27320 100644 --- a/sqlite3/lib/src/sqlite3.dart +++ b/sqlite3/lib/src/sqlite3.dart @@ -1,4 +1,5 @@ import 'database.dart'; +import 'vfs.dart'; /// Provides access to `sqlite3` functions, such as opening new databases. abstract interface class CommonSqlite3 { @@ -25,7 +26,10 @@ abstract interface class CommonSqlite3 { }); /// Opens an in-memory database. - CommonDatabase openInMemory(); + /// + /// The [vfs] option can be used to set the appropriate virtual file system + /// implementation. When null, the default file system will be used. + CommonDatabase openInMemory({String? vfs}); /// Accesses the `sqlite3_temp_directory` variable. /// @@ -34,6 +38,24 @@ abstract interface class CommonSqlite3 { /// /// See also: https://www.sqlite.org/c3ref/temp_directory.html String? tempDirectory; + + /// Registers a custom virtual file system used by this sqlite3 instance to + /// emulate I/O functionality that is not supported through WASM directly. + /// + /// Implementing a suitable [VirtualFileSystem] is a complex task. Users of + /// this package on the web should consider using a package that calls this + /// method for them (like `drift` or `sqflite_common_ffi_web`). + /// For more information on how to implement this, see the readme of the + /// `sqlite3` package for details. + void registerVirtualFileSystem(VirtualFileSystem vfs, + {bool makeDefault = false}); + + /// Unregisters a virtual file system implementation that has been registered + /// with [registerVirtualFileSystem]. + /// + /// sqlite3 is not clear about what happens when this method is called with + /// the file system being in used. Thus, this method should be used with care. + void unregisterVirtualFileSystem(VirtualFileSystem vfs); } /// Version information about the sqlite3 library in use. diff --git a/sqlite3/lib/src/wasm/vfs/utils.dart b/sqlite3/lib/src/utils.dart similarity index 100% rename from sqlite3/lib/src/wasm/vfs/utils.dart rename to sqlite3/lib/src/utils.dart diff --git a/sqlite3/lib/src/vfs.dart b/sqlite3/lib/src/vfs.dart index 7b843bca..7cad5883 100644 --- a/sqlite3/lib/src/vfs.dart +++ b/sqlite3/lib/src/vfs.dart @@ -27,9 +27,6 @@ base class Sqlite3Filename { /// A [virtual filesystem][vfs] used by sqlite3 to access the current I/O /// environment. /// -/// Due to a lack of necessity on other platforms, registering instances of -/// virtual file systems is only supported with the web backend of this package. -/// /// Instead of having an integer return code, file system implementations should /// throw a [VfsException] to signal invalid operations. /// For details on the individual methods, consult the `sqlite3.h` header file diff --git a/sqlite3/lib/src/wasm/bindings.dart b/sqlite3/lib/src/wasm/bindings.dart index ca9026b3..f6245d2d 100644 --- a/sqlite3/lib/src/wasm/bindings.dart +++ b/sqlite3/lib/src/wasm/bindings.dart @@ -76,6 +76,7 @@ final class WasmSqliteBindings extends RawSqliteBindings { return bindings.memory.readString(bindings.sqlite3_sourceid()); } + @override void registerVirtualFileSystem(VirtualFileSystem vfs, int makeDefault) { final name = bindings.allocateZeroTerminated(vfs.name); final id = bindings.callbacks.registerVfs(vfs); @@ -84,6 +85,7 @@ final class WasmSqliteBindings extends RawSqliteBindings { DartCallbacks.sqliteVfsPointer[vfs] = ptr; } + @override void unregisterVirtualFileSystem(VirtualFileSystem vfs) { final ptr = DartCallbacks.sqliteVfsPointer[vfs]; if (ptr == null) { diff --git a/sqlite3/lib/src/wasm/sqlite3.dart b/sqlite3/lib/src/wasm/sqlite3.dart index 498daa11..7718226e 100644 --- a/sqlite3/lib/src/wasm/sqlite3.dart +++ b/sqlite3/lib/src/wasm/sqlite3.dart @@ -5,7 +5,6 @@ import 'dart:typed_data'; import 'package:web/web.dart' as web; import '../implementation/sqlite3.dart'; -import '../vfs.dart'; import 'bindings.dart'; import 'js_interop.dart'; import 'wasm_interop.dart'; @@ -78,27 +77,4 @@ final class WasmSqlite3 extends Sqlite3Implementation { } WasmSqlite3._(WasmBindings bindings) : super(WasmSqliteBindings(bindings)); - - /// Registers a custom virtual file system used by this sqlite3 instance to - /// emulate I/O functionality that is not supported through WASM directly. - /// - /// Implementing a suitable [VirtualFileSystem] is a complex task. Users of - /// this package on the web should consider using a package that calls this - /// method for them (like `drift` or `sqflite_common_ffi_web`). - /// For more information on how to implement this, see the readme of the - /// `sqlite3` package for details. - void registerVirtualFileSystem(VirtualFileSystem vfs, - {bool makeDefault = false}) { - (bindings as WasmSqliteBindings) - .registerVirtualFileSystem(vfs, makeDefault ? 1 : 0); - } - - /// Unregisters a virtual file system implementation that has been registered - /// with [registerVirtualFileSystem]. - /// - /// sqlite3 is not clear about what happens when this method is called with - /// the file system being in used. Thus, this method should be used with care. - void unregisterVirtualFileSystem(VirtualFileSystem vfs) { - (bindings as WasmSqliteBindings).unregisterVirtualFileSystem(vfs); - } } diff --git a/sqlite3/lib/src/wasm/vfs/async_opfs/client.dart b/sqlite3/lib/src/wasm/vfs/async_opfs/client.dart index 5448ad1b..154bbbb7 100644 --- a/sqlite3/lib/src/wasm/vfs/async_opfs/client.dart +++ b/sqlite3/lib/src/wasm/vfs/async_opfs/client.dart @@ -8,7 +8,7 @@ import 'package:path/path.dart' as p; import '../../../constants.dart'; import '../../../vfs.dart'; import '../../js_interop.dart'; -import '../utils.dart'; +import '../../../utils.dart'; import 'sync_channel.dart'; import 'worker.dart'; diff --git a/sqlite3/lib/src/wasm/vfs/indexed_db.dart b/sqlite3/lib/src/wasm/vfs/indexed_db.dart index 186831e9..2cc3a99c 100644 --- a/sqlite3/lib/src/wasm/vfs/indexed_db.dart +++ b/sqlite3/lib/src/wasm/vfs/indexed_db.dart @@ -14,8 +14,8 @@ import 'package:web/web.dart' as web; import '../../constants.dart'; import '../../vfs.dart'; import '../js_interop.dart'; -import 'memory.dart'; -import 'utils.dart'; +import '../../in_memory_vfs.dart'; +import '../../utils.dart'; /// An (asynchronous) file system implementation backed by IndexedDB. /// @@ -46,6 +46,14 @@ class AsynchronousIndexedDbFileSystem { web.IDBDatabase? _database; final String _dbName; + /// Whether to store chunks as [web.Blob]s instead of array buffers. + /// + /// It seems like loading blobs concurrently may be more efficient, but not + /// all browsers support storing blobs in IndexedDB. We support both blobs + /// and array buffers on the read path. For writes, we run a feature detection + /// after opening the file system to determine whether to store blobs. + bool _storeBlobs = true; + AsynchronousIndexedDbFileSystem(this._dbName); bool get _isClosed => _database == null; @@ -79,6 +87,44 @@ class AsynchronousIndexedDbFileSystem { final openFuture = openRequest.completeOrBlocked(); completer.complete(openFuture); _database = await completer.future; + + _storeBlobs = await _supportsStoringBlobs(); + } + + /// Probes whether the IndexedDB implementation supports storing [web.Blob] + /// instances. + /// + /// Safari in private windows does not support storing blobs, but allows + /// storing array buffers directly. Our read paths support reading blobs and + /// array buffers, so we use this to determine which format to use for writes. + Future _supportsStoringBlobs() async { + final transaction = + _database!.transaction([_blocksStore.toJS].toJS, 'readwrite'); + + web.Blob blob; + + try { + final blocks = transaction.objectStore(_blocksStore); + + final request = blocks.add( + web.Blob([Uint8List(4096).buffer.toJS].toJS), + ['test'.toJS].toJS, + ); + final key = await request.complete(); + + blob = await blocks.get(key).complete(); + } on Object { + return false; + } finally { + transaction.abort(); + } + + try { + await blob.byteBuffer(); + return true; + } on Object { + return false; + } } void close() { @@ -163,7 +209,12 @@ class AsynchronousIndexedDbFileSystem { // We can't have an async suspension in here because that would close the // transaction. Launch the reader now and wait for all reads later. readOperations.add(Future.sync(() async { - final data = await (row.value as web.Blob).byteBuffer(); + ByteBuffer data; + if (row.value.instanceOfString('Blob')) { + data = await (row.value as web.Blob).byteBuffer(); + } else { + data = (row.value as JSArrayBuffer).toDart; + } result.setAll(rowOffset, data.asUint8List(0, length)); })); } @@ -189,9 +240,15 @@ class AsynchronousIndexedDbFileSystem { while (await iterator.moveNext()) { final row = iterator.current; - final rowOffset = (row.key! as List)[1] as int; - final blob = row.value as web.Blob; - final dataLength = min(blob.size, file.length - rowOffset); + final key = (row.key as JSArray).toDart; + final rowOffset = (key[1] as JSNumber).toDartInt; + final value = row.value; + final isBlob = value.instanceOfString('Blob'); + final valueSize = isBlob + ? (value as web.Blob).size + : (value as _JSArrayBuffer).byteLength; + + final dataLength = min(valueSize, file.length - rowOffset); if (rowOffset < offset) { // This block starts before the section that we're interested in, so cut @@ -203,7 +260,9 @@ class AsynchronousIndexedDbFileSystem { // Do the reading async because we loose the transaction on the first // suspension. readOperations.add(Future.sync(() async { - final data = await blob.byteBuffer(); + final data = isBlob + ? await (value as web.Blob).byteBuffer() + : (value as _JSArrayBuffer).toDart; target.setRange( 0, @@ -225,7 +284,9 @@ class AsynchronousIndexedDbFileSystem { bytesRead += lengthToCopy; readOperations.add(Future.sync(() async { - final data = await blob.byteBuffer(); + final data = isBlob + ? await (value as web.Blob).byteBuffer() + : (value as _JSArrayBuffer).toDart; target.setAll(startInTarget, data.asUint8List(0, lengthToCopy)); })); @@ -252,15 +313,17 @@ class AsynchronousIndexedDbFileSystem { final cursor = await blocks .openCursor(web.IDBKeyRange.only([fileId.toJS, blockStart.toJS].toJS)) .complete(); - final blob = web.Blob([block.toJS].toJS); + + final value = + _storeBlobs ? web.Blob([block.toJS].toJS) : block.buffer.toJS; if (cursor == null) { // There isn't, let's write a new block await blocks - .put(blob, [fileId.toJS, blockStart.toJS].toJS) + .put(value, [fileId.toJS, blockStart.toJS].toJS) .complete(); } else { - await cursor.update(blob).complete(); + await cursor.update(value).complete(); } } @@ -827,3 +890,8 @@ final class _WriteFileWorkItem extends _IndexedDbWorkItem { ._write(await fileSystem._fileId(path), request); } } + +@JS('ArrayBuffer') +extension type _JSArrayBuffer(JSArrayBuffer _) implements JSArrayBuffer { + external int get byteLength; +} diff --git a/sqlite3/lib/src/wasm/vfs/simple_opfs.dart b/sqlite3/lib/src/wasm/vfs/simple_opfs.dart index d36d1a78..4efd09f1 100644 --- a/sqlite3/lib/src/wasm/vfs/simple_opfs.dart +++ b/sqlite3/lib/src/wasm/vfs/simple_opfs.dart @@ -12,7 +12,7 @@ import 'package:web/web.dart' import '../../constants.dart'; import '../../vfs.dart'; import '../js_interop.dart'; -import 'memory.dart'; +import '../../in_memory_vfs.dart'; @internal enum FileType { diff --git a/sqlite3/lib/wasm.dart b/sqlite3/lib/wasm.dart index 11a71c3d..74c9d22a 100644 --- a/sqlite3/lib/wasm.dart +++ b/sqlite3/lib/wasm.dart @@ -12,13 +12,12 @@ /// As long as this library is marked as experimental, it is not subject to /// semantic versioning. @experimental -library sqlite3.wasm; +library; import 'package:meta/meta.dart'; export 'common.dart'; -export 'src/wasm/vfs/memory.dart' show InMemoryFileSystem; export 'src/wasm/vfs/simple_opfs.dart' show SimpleOpfsFileSystem; export 'src/wasm/vfs/indexed_db.dart' show IndexedDbFileSystem; export 'src/wasm/vfs/async_opfs/client.dart' show WasmVfs; diff --git a/sqlite3/pubspec.yaml b/sqlite3/pubspec.yaml index f9dabeae..4d419ceb 100644 --- a/sqlite3/pubspec.yaml +++ b/sqlite3/pubspec.yaml @@ -1,11 +1,11 @@ name: sqlite3 description: Provides lightweight yet convenient bindings to SQLite by using dart:ffi -version: 2.4.7 +version: 2.5.0 homepage: https://github.com/simolus3/sqlite3.dart/tree/main/sqlite3 issue_tracker: https://github.com/simolus3/sqlite3.dart/issues environment: - sdk: '>=3.4.0 <4.0.0' + sdk: ">=3.5.0 <4.0.0" # This package supports all platforms listed below. platforms: @@ -20,10 +20,11 @@ topics: - sql - database - ffi + - sqlite dependencies: collection: ^1.15.0 - ffi: '>=1.2.1 <3.0.0' + ffi: ">=1.2.1 <3.0.0" meta: ^1.3.0 path: ^1.8.0 web: ^1.0.0 @@ -34,9 +35,9 @@ dev_dependencies: build_daemon: ^4.0.0 build_runner: ^2.1.7 build_web_compilers: ^4.0.3 - ffigen: ^13.0.0 + ffigen: ^16.0.0 http: ^1.2.1 - lints: ^4.0.0 + lints: ^5.0.0 shelf: ^1.4.0 shelf_proxy: ^1.0.2 shelf_static: ^1.1.0 diff --git a/sqlite3/test/common/vfs.dart b/sqlite3/test/common/vfs.dart new file mode 100644 index 00000000..b7432ff9 --- /dev/null +++ b/sqlite3/test/common/vfs.dart @@ -0,0 +1,105 @@ +import 'dart:async'; +import 'dart:typed_data'; + +import 'package:sqlite3/common.dart'; +import 'package:test/test.dart'; + +import 'utils.dart'; + +void testVfs(FutureOr Function() loadSqlite) { + late CommonSqlite3 sqlite3; + + setUpAll(() async => sqlite3 = await loadSqlite()); + + test('smoke check', () { + final vfs = InMemoryFileSystem(name: 'dart'); + sqlite3.registerVirtualFileSystem(vfs); + addTearDown(() => sqlite3.unregisterVirtualFileSystem(vfs)); + + expect(vfs.xAccess('/database', 0), isZero); + var database = sqlite3.open('/database', vfs: 'dart'); + database.execute('CREATE TABLE foo (bar TEXT);'); + database.execute('INSERT INTO foo (bar) VALUES (?)', ['first row']); + expect(vfs.xAccess('/database', 0), isPositive); + + database.dispose(); + database = sqlite3.open('/database', vfs: 'dart'); + expect(database.select('SELECT * FROM foo'), hasLength(1)); + database.dispose(); + }); + + test("can't use vfs after unregistering it", () { + final vfs = InMemoryFileSystem(name: 'dart'); + sqlite3.registerVirtualFileSystem(vfs); + + sqlite3.open('/database', vfs: 'dart').dispose(); + sqlite3.unregisterVirtualFileSystem(vfs); + + expect(() => sqlite3.open('/database', vfs: 'dart'), throwsSqlError(1, 1)); + }); + + test('reports current time', () { + final memory = InMemoryFileSystem(); + final vfs = TestVfs('dart') + ..xOpenDelegate = memory.xOpen + ..xCurrentTimeDelegate = () => DateTime.utc(2024, 11, 19); + sqlite3.registerVirtualFileSystem(vfs); + addTearDown(() => sqlite3.unregisterVirtualFileSystem(vfs)); + + final database = sqlite3.openInMemory(vfs: 'dart'); + addTearDown(database.dispose); + + expect(database.select('SELECT CURRENT_TIMESTAMP AS r'), [ + {'r': '2024-11-19 00:00:00'} + ]); + }); +} + +final class TestVfs extends VirtualFileSystem { + TestVfs(super.name); + + int Function(String, int) xAccessDelegate = (_, __) => 0; + DateTime Function() xCurrentTimeDelegate = DateTime.now; + void Function(String, int)? xDeleteDelegate; + String Function(String) xFullPathNameDelegate = + (_) => throw UnimplementedError(); + XOpenResult Function(Sqlite3Filename path, int flags) xOpenDelegate = + (path, flags) => throw UnimplementedError(); + void Function(Uint8List)? xRandomnessDelegate; + void Function(Duration)? xSleepDelegate; + + @override + int xAccess(String path, int flags) { + return xAccessDelegate(path, flags); + } + + @override + DateTime xCurrentTime() { + return xCurrentTimeDelegate(); + } + + @override + void xDelete(String path, int syncDir) { + return xDeleteDelegate?.call(path, syncDir); + } + + @override + String xFullPathName(String path) { + return xFullPathNameDelegate(path); + } + + @override + XOpenResult xOpen(Sqlite3Filename path, int flags) { + return xOpenDelegate(path, flags); + } + + @override + void xRandomness(Uint8List target) { + return xRandomnessDelegate?.call(target); + } + + @override + void xSleep(Duration duration) { + xSleepDelegate?.call(duration); + } +} diff --git a/sqlite3/test/ffi/vfs_test.dart b/sqlite3/test/ffi/vfs_test.dart new file mode 100644 index 00000000..ae6d7445 --- /dev/null +++ b/sqlite3/test/ffi/vfs_test.dart @@ -0,0 +1,11 @@ +@Tags(['ffi']) +library; + +import 'package:sqlite3/sqlite3.dart'; +import 'package:test/test.dart'; + +import '../common/vfs.dart'; + +void main() { + testVfs(() => sqlite3); +} diff --git a/sqlite3/test/wasm/file_system_test.dart b/sqlite3/test/wasm/file_system_test.dart index d33eec9d..9d461c7f 100644 --- a/sqlite3/test/wasm/file_system_test.dart +++ b/sqlite3/test/wasm/file_system_test.dart @@ -8,6 +8,7 @@ import 'dart:typed_data'; import 'package:sqlite3/wasm.dart'; import 'package:test/test.dart'; +import '../common/vfs.dart'; import 'utils.dart'; const _fsRoot = '/test'; @@ -17,6 +18,10 @@ Future main() async { // this as a fallback. final random = Random(); + group('common test', () { + testVfs(loadSqlite3WithoutVfs); + }); + group('in memory', () { _testWith(() => InMemoryFileSystem(random: random)); }); diff --git a/sqlite3/test/wasm/utils.dart b/sqlite3/test/wasm/utils.dart index b3c33191..a2d9987d 100644 --- a/sqlite3/test/wasm/utils.dart +++ b/sqlite3/test/wasm/utils.dart @@ -3,14 +3,18 @@ import 'dart:math'; import 'package:sqlite3/wasm.dart'; import 'package:test/scaffolding.dart'; -Future loadSqlite3([VirtualFileSystem? defaultVfs]) async { +Future loadSqlite3WithoutVfs() async { final channel = spawnHybridUri('/test/wasm/asset_server.dart'); final port = (await channel.stream.first as double).toInt(); final sqliteWasm = Uri.parse('http://localhost:$port/example/web/sqlite3.wasm'); - final sqlite3 = await WasmSqlite3.loadFromUrl(sqliteWasm); + return await WasmSqlite3.loadFromUrl(sqliteWasm); +} + +Future loadSqlite3([VirtualFileSystem? defaultVfs]) async { + final sqlite3 = await loadSqlite3WithoutVfs(); sqlite3.registerVirtualFileSystem( defaultVfs ?? InMemoryFileSystem( diff --git a/sqlite3_flutter_libs/CHANGELOG.md b/sqlite3_flutter_libs/CHANGELOG.md index 9e7636c4..293a776b 100644 --- a/sqlite3_flutter_libs/CHANGELOG.md +++ b/sqlite3_flutter_libs/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.5.26 + +- Upgrade sqlite to version `3.47.0`. + ## 0.5.25 - Support 16KiB page sizes on Android 15. diff --git a/sqlite3_flutter_libs/android/build.gradle b/sqlite3_flutter_libs/android/build.gradle index 13326cf5..7d80cf2a 100644 --- a/sqlite3_flutter_libs/android/build.gradle +++ b/sqlite3_flutter_libs/android/build.gradle @@ -32,5 +32,5 @@ android { } dependencies { - implementation 'eu.simonbinder:sqlite3-native-library:3.46.1+1' + implementation 'eu.simonbinder:sqlite3-native-library:3.47.0' } diff --git a/sqlite3_flutter_libs/ios/sqlite3_flutter_libs.podspec b/sqlite3_flutter_libs/ios/sqlite3_flutter_libs.podspec index b5b9a019..9e833068 100644 --- a/sqlite3_flutter_libs/ios/sqlite3_flutter_libs.podspec +++ b/sqlite3_flutter_libs/ios/sqlite3_flutter_libs.podspec @@ -17,7 +17,7 @@ A new flutter plugin project. s.public_header_files = 'Classes/**/*.h' s.dependency 'Flutter' - s.dependency 'sqlite3', '~> 3.46.0+1' + s.dependency 'sqlite3', '~> 3.47.0' s.dependency 'sqlite3/fts5' s.dependency 'sqlite3/perf-threadsafe' s.dependency 'sqlite3/rtree' diff --git a/sqlite3_flutter_libs/linux/CMakeLists.txt b/sqlite3_flutter_libs/linux/CMakeLists.txt index 1b375afa..a39a2663 100644 --- a/sqlite3_flutter_libs/linux/CMakeLists.txt +++ b/sqlite3_flutter_libs/linux/CMakeLists.txt @@ -13,13 +13,13 @@ if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0") # We can't really ask users to use a cmake that recent, so there's this if here. FetchContent_Declare( sqlite3 - URL https://sqlite.org/2024/sqlite-autoconf-3460000.tar.gz + URL https://sqlite.org/2024/sqlite-autoconf-3470000.tar.gz DOWNLOAD_EXTRACT_TIMESTAMP NEW ) else() FetchContent_Declare( sqlite3 - URL https://sqlite.org/2024/sqlite-autoconf-3460000.tar.gz + URL https://sqlite.org/2024/sqlite-autoconf-3470000.tar.gz ) endif() FetchContent_MakeAvailable(sqlite3) diff --git a/sqlite3_flutter_libs/macos/sqlite3_flutter_libs.podspec b/sqlite3_flutter_libs/macos/sqlite3_flutter_libs.podspec index cc982e82..a911ca54 100644 --- a/sqlite3_flutter_libs/macos/sqlite3_flutter_libs.podspec +++ b/sqlite3_flutter_libs/macos/sqlite3_flutter_libs.podspec @@ -17,7 +17,7 @@ Pod::Spec.new do |s| s.public_header_files = 'Classes/**/*.h' s.dependency 'FlutterMacOS' - s.dependency 'sqlite3', '~> 3.46.0+1' + s.dependency 'sqlite3', '~> 3.47.0' s.dependency 'sqlite3/fts5' s.dependency 'sqlite3/perf-threadsafe' s.dependency 'sqlite3/rtree' diff --git a/sqlite3_flutter_libs/pubspec.yaml b/sqlite3_flutter_libs/pubspec.yaml index a0e78705..2eeab9bb 100644 --- a/sqlite3_flutter_libs/pubspec.yaml +++ b/sqlite3_flutter_libs/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite3_flutter_libs description: Flutter plugin to include native sqlite3 libraries with your app -version: 0.5.25 +version: 0.5.26 homepage: https://github.com/simolus3/sqlite3.dart/tree/main/sqlite3_flutter_libs issue_tracker: https://github.com/simolus3/sqlite3.dart/issues diff --git a/sqlite3_flutter_libs/windows/CMakeLists.txt b/sqlite3_flutter_libs/windows/CMakeLists.txt index 956bf1f2..259ffb9f 100644 --- a/sqlite3_flutter_libs/windows/CMakeLists.txt +++ b/sqlite3_flutter_libs/windows/CMakeLists.txt @@ -29,13 +29,13 @@ if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0") # We can't really ask users to use a cmake that recent, so there's this if here. FetchContent_Declare( sqlite3 - URL https://sqlite.org/2024/sqlite-autoconf-3460000.tar.gz + URL https://sqlite.org/2024/sqlite-autoconf-3470000.tar.gz DOWNLOAD_EXTRACT_TIMESTAMP NEW ) else() FetchContent_Declare( sqlite3 - URL https://sqlite.org/2024/sqlite-autoconf-3460000.tar.gz + URL https://sqlite.org/2024/sqlite-autoconf-3470000.tar.gz ) endif() FetchContent_MakeAvailable(sqlite3) diff --git a/sqlite3_test/.gitignore b/sqlite3_test/.gitignore new file mode 100644 index 00000000..3cceda55 --- /dev/null +++ b/sqlite3_test/.gitignore @@ -0,0 +1,7 @@ +# https://dart.dev/guides/libraries/private-files +# Created by `dart pub` +.dart_tool/ + +# Avoid committing pubspec.lock for library packages; see +# https://dart.dev/guides/libraries/private-files#pubspeclock. +pubspec.lock diff --git a/sqlite3_test/CHANGELOG.md b/sqlite3_test/CHANGELOG.md new file mode 100644 index 00000000..a0712a79 --- /dev/null +++ b/sqlite3_test/CHANGELOG.md @@ -0,0 +1,3 @@ +## 0.1.0 + +- Initial version. diff --git a/sqlite3_test/README.md b/sqlite3_test/README.md new file mode 100644 index 00000000..5cf99e04 --- /dev/null +++ b/sqlite3_test/README.md @@ -0,0 +1,76 @@ +This package provides utilities for accessing SQLite databases in Dart tests. + +## Features + +Given that SQLite has no external dependencies and runs in the process of your +app, it can easily be used in unit tests (avoiding the hassle of writing mocks +for your database and repositories). + +However, being a C library, SQLite is unaware of other Dart utilities typically +used in tests (like a fake time with `package:clock` or a custom file system +based on `package:file`). +When your database queries depend on `CURRENT_TIMESTAMP`, this makes it hard +to reliably test them as `clock.now()` and `CURRENT_TIMESTAMP` would report +different values. + +As a solution, this small package makes SQLite easier to integrate into your +tests by providing a [VFS](https://sqlite.org/vfs.html) that will: + +1. Make `CURRENT_TIME`, `CURRENT_DATE` and `CURRENT_TIMESTAMP` reflect the time + returned by `package:clock`. +2. For IO operations, allow providing a `FileSystem` from `package:file`. This + includes custom implementations and the default one respecting + `IOOverrides`. + +## Usage + +This package is intended to be used in tests, so begin by adding a dev +dependency on it: + +``` +$ dart pub add --dev sqlite3_test +``` + +You can then use it in tests by creating an instance of `TestSqliteFileSystem` +for your databases: + +```dart +import 'package:fake_async/fake_async.dart'; +import 'package:sqlite3/sqlite3.dart'; +import 'package:sqlite3_test/sqlite3_test.dart'; +import 'package:file/local.dart'; +import 'package:test/test.dart'; + +void main() { + late TestSqliteFileSystem vfs; + + setUpAll(() { + vfs = TestSqliteFileSystem(fs: const LocalFileSystem()); + sqlite3.registerVirtualFileSystem(vfs); + }); + tearDownAll(() => sqlite3.unregisterVirtualFileSystem(vfs)); + + test('my test depending on database time', () { + final database = sqlite3.openInMemory(vfs: vfs.name); + addTearDown(database.dispose); + + // The VFS uses package:clock to get the current time, which can be + // overridden for tests: + final moonLanding = DateTime.utc(1969, 7, 20, 20, 18, 04); + FakeAsync(initialTime: moonLanding).run((_) { + final row = database.select('SELECT unixepoch(current_timestamp)').first; + + expect(row.columnAt(0), -14182916); + }); + }); +} +``` + +## Limitations + +The layer of indirection through Dart will likely make your databases slower. +For this reason, this package is intended to be used in tests (as the overhead +is not a problem there). + +Also, note that `TestSqliteFileSystem` cannot be used with WAL databases as the +file system does not implement memory-mapped IO. diff --git a/sqlite3_test/analysis_options.yaml b/sqlite3_test/analysis_options.yaml new file mode 100644 index 00000000..572dd239 --- /dev/null +++ b/sqlite3_test/analysis_options.yaml @@ -0,0 +1 @@ +include: package:lints/recommended.yaml diff --git a/sqlite3_test/example/sqlite3_test_example.dart b/sqlite3_test/example/sqlite3_test_example.dart new file mode 100644 index 00000000..a6629e51 --- /dev/null +++ b/sqlite3_test/example/sqlite3_test_example.dart @@ -0,0 +1,29 @@ +import 'package:fake_async/fake_async.dart'; +import 'package:sqlite3/sqlite3.dart'; +import 'package:sqlite3_test/sqlite3_test.dart'; +import 'package:file/local.dart'; +import 'package:test/test.dart'; + +void main() { + late TestSqliteFileSystem vfs; + + setUpAll(() { + vfs = TestSqliteFileSystem(fs: const LocalFileSystem()); + sqlite3.registerVirtualFileSystem(vfs); + }); + tearDownAll(() => sqlite3.unregisterVirtualFileSystem(vfs)); + + test('my test depending on database time', () { + final database = sqlite3.openInMemory(vfs: vfs.name); + addTearDown(database.dispose); + + // The VFS uses package:clock to get the current time, which can be + // overridden for tests: + final moonLanding = DateTime.utc(1969, 7, 20, 20, 18, 04); + FakeAsync(initialTime: moonLanding).run((_) { + final row = database.select('SELECT unixepoch(current_timestamp)').first; + + expect(row.columnAt(0), -14182916); + }); + }); +} diff --git a/sqlite3_test/lib/sqlite3_test.dart b/sqlite3_test/lib/sqlite3_test.dart new file mode 100644 index 00000000..252877eb --- /dev/null +++ b/sqlite3_test/lib/sqlite3_test.dart @@ -0,0 +1,188 @@ +/// Provides a virtual filesystem implementation for SQLite based on the `file` +/// and `clock` packages. +/// +/// This makes it easier to use SQLite in tests, as SQL constructs like +/// `CURRENT_TIMESTAMP` will reflect the fake time of `package:clock`, allowing +/// SQL logic relying on time to be tested reliably. Additionally, using +/// `dart:clock` allows testing the IO behavior of SQLite databases if +/// necessary. +library; + +import 'dart:typed_data'; + +import 'package:clock/clock.dart'; +import 'package:file/file.dart'; +import 'package:sqlite3/common.dart'; + +final class TestSqliteFileSystem extends BaseVirtualFileSystem { + static int _counter = 0; + + final FileSystem _fs; + Directory? _createdTmp; + int _tmpFileCounter = 0; + + TestSqliteFileSystem({required FileSystem fs, String? name}) + : _fs = fs, + super(name: name ?? 'dart-test-vfs-${_counter++}'); + + Directory get _tempDirectory { + return _createdTmp ??= + _fs.systemTempDirectory.createTempSync('dart-sqlite3-test'); + } + + @override + int xAccess(String path, int flags) { + switch (flags) { + case 0: + // Exists + return _fs.typeSync(path) == FileSystemEntityType.file ? 1 : 0; + default: + // Check readable and writable + try { + final file = _fs.file(path); + file.openSync(mode: FileMode.write).closeSync(); + return 1; + } on IOException { + return 0; + } + } + } + + @override + DateTime xCurrentTime() { + return clock.now(); + } + + @override + void xDelete(String path, int syncDir) { + _fs.file(path).deleteSync(); + } + + @override + String xFullPathName(String path) { + return _fs.path.absolute(path); + } + + @override + XOpenResult xOpen(Sqlite3Filename path, int flags) { + final fsPath = path.path ?? + _tempDirectory.childFile((_tmpFileCounter++).toString()).absolute.path; + final type = _fs.typeSync(fsPath); + + if (type != FileSystemEntityType.notFound && + type != FileSystemEntityType.file) { + throw VfsException(ErrorCodes.EINVAL); + } + + if (flags & SqlFlag.SQLITE_OPEN_EXCLUSIVE != 0 && + type != FileSystemEntityType.notFound) { + throw VfsException(ErrorCodes.EEXIST); + } + if (flags & SqlFlag.SQLITE_OPEN_CREATE != 0 && + type == FileSystemEntityType.notFound) { + _fs.file(fsPath).createSync(); + } + + final deleteOnClose = flags & SqlFlag.SQLITE_OPEN_DELETEONCLOSE != 0; + final readonly = flags & SqlFlag.SQLITE_OPEN_READONLY != 0; + final vsFile = _fs.file(fsPath); + final file = + vsFile.openSync(mode: readonly ? FileMode.read : FileMode.write); + + return ( + file: _TestFile(vsFile, file, deleteOnClose), + outFlags: readonly ? SqlFlag.SQLITE_OPEN_READONLY : 0, + ); + } + + @override + void xSleep(Duration duration) {} +} + +final class _TestFile implements VirtualFileSystemFile { + final File _path; + final RandomAccessFile _file; + final bool _deleteOnClose; + int _lockLevel = SqlFileLockingLevels.SQLITE_LOCK_NONE; + + _TestFile(this._path, this._file, this._deleteOnClose); + + @override + void xClose() { + _file.closeSync(); + if (_deleteOnClose) { + _path.deleteSync(); + } + } + + @override + int get xDeviceCharacteristics => 0; + + @override + int xFileSize() => _file.lengthSync(); + + @override + void xRead(Uint8List target, int fileOffset) { + _file.setPositionSync(fileOffset); + final bytesRead = _file.readIntoSync(target); + if (bytesRead < target.length) { + target.fillRange(bytesRead, target.length, 0); + throw VfsException(SqlExtendedError.SQLITE_IOERR_SHORT_READ); + } + } + + @override + void xSync(int flags) { + _file.flushSync(); + } + + @override + void xTruncate(int size) { + _file.truncateSync(size); + } + + @override + void xWrite(Uint8List buffer, int fileOffset) { + _file + ..setPositionSync(fileOffset) + ..writeFromSync(buffer); + } + + @override + int xCheckReservedLock() { + // RandomAccessFile doesn't appear to expose information on whether another + // process is holding locks. + return _lockLevel > SqlFileLockingLevels.SQLITE_LOCK_NONE ? 1 : 0; + } + + @override + void xLock(int mode) { + if (_lockLevel >= mode) { + return; + } + + if (_lockLevel != SqlFileLockingLevels.SQLITE_LOCK_NONE) { + // We want to upgrade our lock, which we do by releasing it and then + // re-obtaining it. + _file.unlockSync(); + _lockLevel = SqlFileLockingLevels.SQLITE_LOCK_NONE; + } + + final exclusive = mode > SqlFileLockingLevels.SQLITE_LOCK_SHARED; + _file.lockSync( + exclusive ? FileLock.blockingExclusive : FileLock.blockingShared); + _lockLevel = mode; + } + + @override + void xUnlock(int mode) { + if (_lockLevel < mode) { + return; + } + + _file.unlockSync(); + if (mode != SqlFileLockingLevels.SQLITE_LOCK_NONE) { + return xLock(mode); + } + } +} diff --git a/sqlite3_test/pubspec.yaml b/sqlite3_test/pubspec.yaml new file mode 100644 index 00000000..12f1517e --- /dev/null +++ b/sqlite3_test/pubspec.yaml @@ -0,0 +1,26 @@ +name: sqlite3_test +description: Integration of fake clocks and other test utilities for SQLite databases. +version: 0.1.0 +homepage: https://github.com/simolus3/sqlite3.dart/tree/main/sqlite3_test +repository: https://github.com/simolus3/sqlite3.dart +topics: + - database + - sqlite + +environment: + sdk: ^3.5.4 + +dependencies: + clock: ^1.1.2 + file: ^7.0.1 + sqlite3: ^2.5.0 + +dev_dependencies: + fake_async: ^1.3.2 + lints: ^4.0.0 + test: ^1.24.0 + test_descriptor: ^2.0.1 + +dependency_overrides: + sqlite3: + path: ../sqlite3 diff --git a/sqlite3_test/test/sqlite3_test_test.dart b/sqlite3_test/test/sqlite3_test_test.dart new file mode 100644 index 00000000..a197b443 --- /dev/null +++ b/sqlite3_test/test/sqlite3_test_test.dart @@ -0,0 +1,65 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:fake_async/fake_async.dart'; +import 'package:sqlite3/sqlite3.dart'; +import 'package:sqlite3_test/sqlite3_test.dart'; +import 'package:test/test.dart'; +import 'package:test_descriptor/test_descriptor.dart' as d; +import 'package:file/local.dart'; + +void main() { + late TestSqliteFileSystem vfs; + + setUp(() { + vfs = TestSqliteFileSystem(fs: const LocalFileSystem()); + sqlite3.registerVirtualFileSystem(vfs, makeDefault: false); + }); + tearDown(() => sqlite3.unregisterVirtualFileSystem(vfs)); + + Database withDatabase(Database db) { + addTearDown(db.dispose); + return db; + } + + Database inMemory() => withDatabase(sqlite3.openInMemory(vfs: vfs.name)); + Database onDisk(String path) => + withDatabase(sqlite3.open(path, vfs: vfs.name)); + + test('reports fake time', () { + final moonLanding = DateTime.utc(1969, 7, 20, 20, 18, 04); + + FakeAsync(initialTime: moonLanding).run((async) { + final db = inMemory(); + + expect(db.select('SELECT current_time AS r'), [ + {'r': '20:18:04'} + ]); + expect(db.select('SELECT current_date AS r'), [ + {'r': '1969-07-20'} + ]); + expect(db.select('SELECT current_timestamp AS r'), [ + {'r': '1969-07-20 20:18:04'} + ]); + }); + }); + + test('use fake cwd from io overrides', () async { + await d.dir('foo').create(); + final cwd = Directory(d.path('foo')); + + IOOverrides.runZoned(() { + final db = onDisk('test.db'); + db.execute('CREATE TABLE foo (bar);'); + }, getCurrentDirectory: () => cwd); + + await d.dir('foo', [ + d.FileDescriptor.binaryMatcher( + 'test.db', + predicate((bytes) { + final firstBytes = (bytes as List).sublist(0, 15); + return utf8.decode(firstBytes) == 'SQLite format 3'; + }, 'starts with sqlite header')), + ]).validate(); + }); +} diff --git a/sqlite3_web/CHANGELOG.md b/sqlite3_web/CHANGELOG.md index 490494e5..cddf1820 100644 --- a/sqlite3_web/CHANGELOG.md +++ b/sqlite3_web/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.2.0 + +- Make `FileSystem` implementation functional, add `FileSystem.flush()`. + ## 0.1.3 - Support latest version of `package:web`. diff --git a/sqlite3_web/lib/src/client.dart b/sqlite3_web/lib/src/client.dart index bbc106e7..c21f0dd8 100644 --- a/sqlite3_web/lib/src/client.dart +++ b/sqlite3_web/lib/src/client.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'dart:js_interop'; import 'dart:js_interop_unsafe'; +import 'dart:typed_data'; import 'package:sqlite3/common.dart'; import 'package:web/web.dart' @@ -92,12 +93,12 @@ final class RemoteDatabase implements Database { } @override - FileSystem get fileSystem => throw UnimplementedError(); + late final FileSystem fileSystem = RemoteFileSystem(this); @override Future get lastInsertRowId async { final result = await select('select last_insert_rowid();'); - return result.single[0] as int; + return result.single.columnAt(0) as int; } @override @@ -128,7 +129,7 @@ final class RemoteDatabase implements Database { @override Future get userVersion async { final result = await select('pragma user_version;'); - return result.single[0] as int; + return result.single.columnAt(0) as int; } @override @@ -142,6 +143,66 @@ final class RemoteDatabase implements Database { } } +final class RemoteFileSystem implements FileSystem { + final RemoteDatabase database; + + RemoteFileSystem(this.database); + + @override + Future exists(FileType type) async { + final response = await database.connection.sendRequest( + FileSystemExistsQuery( + databaseId: database.databaseId, + fsType: type, + requestId: 0, + ), + MessageType.simpleSuccessResponse, + ); + + return (response.response as JSBoolean).toDart; + } + + @override + Future flush() async { + await database.connection.sendRequest( + FileSystemFlushRequest(databaseId: database.databaseId, requestId: 0), + MessageType.simpleSuccessResponse, + ); + } + + @override + Future readFile(FileType type) async { + final response = await database.connection.sendRequest( + FileSystemAccess( + databaseId: database.databaseId, + requestId: 0, + buffer: null, + fsType: type, + ), + MessageType.simpleSuccessResponse, + ); + + final buffer = (response.response as JSArrayBuffer); + return buffer.toDart.asUint8List(); + } + + @override + Future writeFile(FileType type, Uint8List content) async { + // We need to copy since we're about to transfer contents over + final copy = Uint8List(content.length)..setAll(0, content); + + await database.connection.sendRequest( + FileSystemAccess( + databaseId: database.databaseId, + requestId: 0, + buffer: copy.buffer.toJS, + fsType: type, + ), + MessageType.simpleSuccessResponse, + ); + } +} + final class WorkerConnection extends ProtocolChannel { final StreamController notifications = StreamController.broadcast(); @@ -330,8 +391,8 @@ final class DatabaseClient implements WebSqlite { } @override - Future connect( - String name, StorageMode type, AccessMode access) async { + Future connect(String name, StorageMode type, AccessMode access, + {bool onlyOpenVfs = false}) async { await startWorkers(); WorkerConnection connection; @@ -360,6 +421,7 @@ final class DatabaseClient implements WebSqlite { wasmUri: wasmUri, databaseName: name, storageMode: type.resolveToVfs(shared), + onlyOpenVfs: onlyOpenVfs, ), MessageType.simpleSuccessResponse, ); @@ -370,7 +432,8 @@ final class DatabaseClient implements WebSqlite { } @override - Future connectToRecommended(String name) async { + Future connectToRecommended(String name, + {bool onlyOpenVfs = false}) async { final probed = await runFeatureDetection(databaseName: name); // If we have an existing database in storage, we want to keep using that @@ -399,7 +462,8 @@ final class DatabaseClient implements WebSqlite { final (storage, access) = availableImplementations.firstOrNull ?? (StorageMode.inMemory, AccessMode.inCurrentContext); - final database = await connect(name, storage, access); + final database = + await connect(name, storage, access, onlyOpenVfs: onlyOpenVfs); return ConnectToRecommendedResult( database: database, diff --git a/sqlite3_web/lib/src/database.dart b/sqlite3_web/lib/src/database.dart index fb19ae7e..15e78890 100644 --- a/sqlite3_web/lib/src/database.dart +++ b/sqlite3_web/lib/src/database.dart @@ -162,11 +162,27 @@ abstract class WebSqlite { /// Connects to a database identified by its [name] stored under [type] and /// accessed via the given [access] mode. - Future connect(String name, StorageMode type, AccessMode access); + /// + /// When [onlyOpenVfs] is enabled, only the underlying file system for the + /// database is initialized before [connect] returns. By default, the database + /// will also be opened in [connect]. Otherwise, the database will be opened + /// on the worker when it's first used. + /// Only opening the VFS can be used to, for instance, check if the database + /// already exists and to initialize it manually if it doesn't. + Future connect(String name, StorageMode type, AccessMode access, + {bool onlyOpenVfs = false}); /// Starts a feature detection via [runFeatureDetection] and then [connect]s /// to the best database available. - Future connectToRecommended(String name); + /// + /// When [onlyOpenVfs] is enabled, only the underlying file system for the + /// database is initialized before [connect] returns. By default, the database + /// will also be opened in [connect]. Otherwise, the database will be opened + /// on the worker when it's first used. + /// Only opening the VFS can be used to, for instance, check if the database + /// already exists and to initialize it manually if it doesn't. + Future connectToRecommended(String name, + {bool onlyOpenVfs = false}); /// Entrypoints for workers hosting datbases. static void workerEntrypoint({ diff --git a/sqlite3_web/lib/src/protocol.dart b/sqlite3_web/lib/src/protocol.dart index 79e2fa7b..90f58310 100644 --- a/sqlite3_web/lib/src/protocol.dart +++ b/sqlite3_web/lib/src/protocol.dart @@ -22,6 +22,7 @@ enum MessageType { runQuery(), fileSystemExists(), fileSystemAccess(), + fileSystemFlush(), connect(), startFileSystemServer(), updateRequest(), @@ -52,6 +53,7 @@ class _UniqueFieldNames { static const id = 'i'; static const updateKind = 'k'; static const tableNames = 'n'; + static const onlyOpenVfs = 'o'; static const parameters = 'p'; static const storageMode = 's'; static const sql = 's'; // not used in same message @@ -86,6 +88,7 @@ sealed class Message { MessageType.runQuery => RunQuery.deserialize(object), MessageType.fileSystemExists => FileSystemExistsQuery.deserialize(object), MessageType.fileSystemAccess => FileSystemAccess.deserialize(object), + MessageType.fileSystemFlush => FileSystemFlushRequest.deserialize(object), MessageType.connect => ConnectRequest.deserialize(object), MessageType.closeDatabase => CloseDatabase.deserialize(object), MessageType.openAdditionalConnection => @@ -199,12 +202,14 @@ final class OpenRequest extends Request { final String databaseName; final FileSystemImplementation storageMode; + final bool onlyOpenVfs; OpenRequest({ required super.requestId, required this.wasmUri, required this.databaseName, required this.storageMode, + required this.onlyOpenVfs, }); factory OpenRequest.deserialize(JSObject object) { @@ -215,6 +220,9 @@ final class OpenRequest extends Request { wasmUri: Uri.parse((object[_UniqueFieldNames.wasmUri] as JSString).toDart), requestId: object.requestId, + onlyOpenVfs: + // The onlyOpenVfs field was not set in earlier clients. + (object[_UniqueFieldNames.onlyOpenVfs] as JSBoolean?)?.toDart == true, ); } @@ -227,6 +235,7 @@ final class OpenRequest extends Request { object[_UniqueFieldNames.databaseName] = databaseName.toJS; object[_UniqueFieldNames.storageMode] = storageMode.toJS; object[_UniqueFieldNames.wasmUri] = wasmUri.toString().toJS; + object[_UniqueFieldNames.onlyOpenVfs] = onlyOpenVfs.toJS; } } @@ -348,6 +357,24 @@ final class FileSystemExistsQuery extends Request { } } +/// Requests the worker to flush the file system for a database. +final class FileSystemFlushRequest extends Request { + @override + MessageType get type => MessageType.fileSystemFlush; + + FileSystemFlushRequest({ + required super.databaseId, + required super.requestId, + }); + + factory FileSystemFlushRequest.deserialize(JSObject object) { + return FileSystemFlushRequest( + databaseId: object.databaseId, + requestId: object.requestId, + ); + } +} + /// Read or write to files of an opened database. /// /// For reads, other side will respond with a [SimpleSuccessResponse] containing @@ -385,8 +412,6 @@ final class FileSystemAccess extends Request { object[_UniqueFieldNames.buffer] = buffer; object[_UniqueFieldNames.fileType] = fsType.index.toJS; - // false positive? dart2js seems to emit a null check as it should - // ignore: pattern_never_matches_value_type if (buffer case final buffer?) { transferred.add(buffer); } @@ -462,6 +487,9 @@ final class OpenAdditonalConnection extends Request { MessageType get type => MessageType.openAdditionalConnection; } +@JS('ArrayBuffer') +external JSFunction get _arrayBufferConstructor; + final class SimpleSuccessResponse extends Response { final JSAny? response; @@ -481,6 +509,10 @@ final class SimpleSuccessResponse extends Response { void serialize(JSObject object, List transferred) { super.serialize(object, transferred); object[_UniqueFieldNames.responseData] = response; + + if (response.instanceof(_arrayBufferConstructor)) { + transferred.add(response as JSObject); + } } } diff --git a/sqlite3_web/lib/src/types.dart b/sqlite3_web/lib/src/types.dart index 883f710a..e3b21b27 100644 --- a/sqlite3_web/lib/src/types.dart +++ b/sqlite3_web/lib/src/types.dart @@ -75,13 +75,21 @@ final class RemoteException implements Exception { } } +/// A virtual file system used by a worker to persist database files. abstract class FileSystem { - StorageMode get storage; - String get databaseName; - + /// Returns whether a database file identified by its [type] exists. Future exists(FileType type); + + /// Reads the database file (or its journal). Future readFile(FileType type); + + /// Replaces the database file (or its journal), creating the virtual file if + /// it doesn't exist. Future writeFile(FileType type, Uint8List content); + + /// If the file system hosting the database in the worker is not synchronous, + /// flushes pending writes. + Future flush(); } /// An enumeration of features not supported by the current browsers. diff --git a/sqlite3_web/lib/src/worker.dart b/sqlite3_web/lib/src/worker.dart index d8bab3f0..84411bc8 100644 --- a/sqlite3_web/lib/src/worker.dart +++ b/sqlite3_web/lib/src/worker.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'dart:js_interop'; import 'dart:js_interop_unsafe'; +import 'dart:typed_data'; import 'package:sqlite3/wasm.dart'; import 'package:stream_channel/stream_channel.dart'; import 'package:web/web.dart' @@ -21,6 +22,7 @@ import 'database.dart'; import 'channel.dart'; import 'protocol.dart'; import 'shared.dart'; +import 'types.dart'; sealed class WorkerEnvironment { WorkerEnvironment._(); @@ -180,9 +182,12 @@ final class _ClientConnection extends ProtocolChannel try { database = _runner.findDatabase(request.databaseName, request.storageMode); - await database.opened; + + await (request.onlyOpenVfs ? database.vfs : database.opened); + connectionDatabase = _ConnectionDatabase(database); _openedDatabases.add(connectionDatabase); + return SimpleSuccessResponse( response: database.id.toJS, requestId: request.requestId); } catch (e) { @@ -239,10 +244,44 @@ final class _ClientConnection extends ProtocolChannel await database.close(); return SimpleSuccessResponse( response: null, requestId: request.requestId); - case FileSystemExistsQuery(): - throw UnimplementedError(); - case FileSystemAccess(): - throw UnimplementedError(); + case FileSystemFlushRequest(): + if (await database?.database.vfs case IndexedDbFileSystem idb) { + await idb.flush(); + } + + return SimpleSuccessResponse( + response: null, requestId: request.requestId); + case FileSystemExistsQuery(:final fsType): + final vfs = await database!.database.vfs; + final exists = vfs.xAccess(fsType.pathInVfs, 0) == 1; + + return SimpleSuccessResponse( + response: exists.toJS, requestId: request.requestId); + case FileSystemAccess(:final buffer, :final fsType): + final vfs = await database!.database.vfs; + final file = vfs + .xOpen( + Sqlite3Filename(fsType.pathInVfs), SqlFlag.SQLITE_OPEN_CREATE) + .file; + + try { + if (buffer != null) { + final asDartBuffer = buffer.toDart; + file.xTruncate(asDartBuffer.lengthInBytes); + file.xWrite(asDartBuffer.asUint8List(), 0); + + return SimpleSuccessResponse( + response: null, requestId: request.requestId); + } else { + final buffer = Uint8List(file.xFileSize()); + file.xRead(buffer, 0); + + return SimpleSuccessResponse( + response: buffer.buffer.toJS, requestId: request.requestId); + } + } finally { + file.xClose(); + } } } @@ -268,6 +307,13 @@ final class _ClientConnection extends ProtocolChannel } } +extension on FileType { + String get pathInVfs => switch (this) { + FileType.database => '/database', + FileType.journal => '/database-journal', + }; +} + final class DatabaseState { final WorkerRunner runner; final int id; @@ -276,11 +322,12 @@ final class DatabaseState { int refCount = 1; Future? _database; + Future? _openVfs; + VirtualFileSystem? _resolvedVfs; /// Runs additional async work, such as flushing the VFS to IndexedDB when /// the database is closed. FutureOr Function()? closeHandler; - VirtualFileSystem? vfs; DatabaseState( {required this.id, @@ -288,11 +335,10 @@ final class DatabaseState { required this.name, required this.mode}); - Future get opened async { - final database = _database ??= Future.sync(() async { - final sqlite3 = await runner._sqlite3!; - final vfsName = 'vfs-web-$id'; + String get vfsName => 'vfs-web-$id'; + Future get vfs async { + await (_openVfs ??= Future.sync(() async { switch (mode) { case FileSystemImplementation.opfsLocks: final options = WasmVfs.createOptions(root: pathForOpfs(name)); @@ -304,22 +350,31 @@ final class DatabaseState { await EventStreamProviders.messageEvent.forTarget(worker).first; final wasmVfs = - vfs = WasmVfs(workerOptions: options, vfsName: vfsName); + _resolvedVfs = WasmVfs(workerOptions: options, vfsName: vfsName); closeHandler = wasmVfs.close; case FileSystemImplementation.opfsShared: - final simple = vfs = await SimpleOpfsFileSystem.loadFromStorage( - pathForOpfs(name), - vfsName: vfsName); + final simple = _resolvedVfs = + await SimpleOpfsFileSystem.loadFromStorage(pathForOpfs(name), + vfsName: vfsName); closeHandler = simple.close; case FileSystemImplementation.indexedDb: - final idb = vfs = + final idb = _resolvedVfs = await IndexedDbFileSystem.open(dbName: name, vfsName: vfsName); closeHandler = idb.close; case FileSystemImplementation.inMemory: - vfs = InMemoryFileSystem(name: vfsName); + _resolvedVfs = InMemoryFileSystem(name: vfsName); } + })); + + return _resolvedVfs!; + } + + Future get opened async { + final database = _database ??= Future.sync(() async { + final sqlite3 = await runner._sqlite3!; + final fileSystem = await vfs; - sqlite3.registerVirtualFileSystem(vfs!); + sqlite3.registerVirtualFileSystem(fileSystem); return await runner._controller.openDatabase( sqlite3, // We're currently using /database as the in-VFS path. This is because @@ -346,7 +401,7 @@ final class DatabaseState { final database = await _database!; database.database.dispose(); - if (vfs case final vfs?) { + if (_resolvedVfs case final vfs?) { sqlite3.unregisterVirtualFileSystem(vfs); } diff --git a/sqlite3_web/pubspec.yaml b/sqlite3_web/pubspec.yaml index 08b6ca0f..e66cb69a 100644 --- a/sqlite3_web/pubspec.yaml +++ b/sqlite3_web/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite3_web description: Utilities to simplify accessing sqlite3 on the web, with automated feature detection. -version: 0.1.3 +version: 0.2.0 homepage: https://github.com/simolus3/sqlite3.dart/tree/main/sqlite3_web repository: https://github.com/simolus3/sqlite3.dart @@ -13,7 +13,7 @@ dependencies: web: ^1.0.0 dev_dependencies: - lints: ^2.1.0 + lints: ^5.0.0 test: ^1.25.5 build_web_compilers: ^4.0.9 build_runner: ^2.4.8 diff --git a/sqlite3_web/test/integration_test.dart b/sqlite3_web/test/integration_test.dart index 659108c2..10322a28 100644 --- a/sqlite3_web/test/integration_test.dart +++ b/sqlite3_web/test/integration_test.dart @@ -1,3 +1,4 @@ +import 'dart:async'; import 'dart:io'; import 'package:sqlite3_web/src/types.dart'; @@ -82,9 +83,24 @@ void main() { group(browser.name, () { late Process driverProcess; late TestWebDriver driver; + var isStoppingProcess = false; + final processStopped = Completer(); + + setUpAll(() async { + final process = driverProcess = await browser.spawnDriver(); + process.exitCode.then((code) { + if (!isStoppingProcess) { + throw 'Webdriver stopped (code $code) before tearing down tests.'; + } - setUpAll(() async => driverProcess = await browser.spawnDriver()); - tearDownAll(() => driverProcess.kill()); + processStopped.complete(); + }); + }); + tearDownAll(() { + isStoppingProcess = true; + driverProcess.kill(); + return processStopped.future; + }); setUp(() async { final rawDriver = await createDriver( @@ -103,9 +119,12 @@ void main() { }, ); - rawDriver.logs.get(LogType.browser).listen((entry) { - print('[console]: ${entry.message}'); - }); + // logs.get() isn't supported on Firefox + if (browser != Browser.firefox) { + rawDriver.logs.get(LogType.browser).listen((entry) { + print('[console]: ${entry.message}'); + }); + } driver = TestWebDriver(server, rawDriver); await driver.driver.get('http://localhost:8080/'); @@ -138,11 +157,19 @@ void main() { for (final (storage, access) in browser.availableImplementations) { test('$storage through $access', () async { - await driver.openDatabase((storage, access)); + await driver.openDatabase( + implementation: (storage, access), + onlyOpenVfs: true, + ); + await driver.assertFile(false); + await driver.execute('CREATE TABLE foo (bar TEXT);'); expect(await driver.countUpdateEvents(), 0); await driver.execute("INSERT INTO foo (bar) VALUES ('hello');"); expect(await driver.countUpdateEvents(), 1); + + expect(await driver.assertFile(true), isPositive); + await driver.flush(); }); } }); diff --git a/sqlite3_web/tool/server.dart b/sqlite3_web/tool/server.dart index a20ec2d2..6ea74e53 100644 --- a/sqlite3_web/tool/server.dart +++ b/sqlite3_web/tool/server.dart @@ -136,15 +136,18 @@ class TestWebDriver { ); } - Future<(StorageMode, AccessMode)> openDatabase( - [(StorageMode, AccessMode)? implementation]) async { + Future<(StorageMode, AccessMode)> openDatabase({ + (StorageMode, AccessMode)? implementation, + bool onlyOpenVfs = false, + }) async { final desc = switch (implementation) { null => null, (var storage, var access) => '${storage.name}:${access.name}' }; + final method = onlyOpenVfs ? 'open_only_vfs' : 'open'; final res = await driver - .executeAsync('open(arguments[0], arguments[1])', [desc]) as String?; + .executeAsync('$method(arguments[0], arguments[1])', [desc]) as String?; // This returns the storage/access mode actually chosen. final split = res!.split(':'); @@ -175,4 +178,27 @@ class TestWebDriver { throw 'test_second failed! More information may be available in the console.'; } } + + Future assertFile(bool shouldExist) async { + final res = await driver.executeAsync( + 'assert_file(arguments[0], arguments[1])', [shouldExist.toString()]); + res!; + + if (res == false) { + throw 'assertFile failed! Expected $shouldExist, match return was $res'; + } + + if (res is int) { + return res; + } else { + return null; + } + } + + Future flush() async { + final result = await driver.executeAsync('flush("", arguments[0])', []); + if (result != true) { + throw 'flush() failed: $result'; + } + } } diff --git a/sqlite3_web/web/main.dart b/sqlite3_web/web/main.dart index 6e63df33..04328335 100644 --- a/sqlite3_web/web/main.dart +++ b/sqlite3_web/web/main.dart @@ -13,6 +13,7 @@ WebSqlite? webSqlite; Database? database; int updates = 0; +bool listeningForUpdates = false; void main() { _addCallbackForWebDriver('detectImplementations', _detectImplementations); @@ -21,9 +22,11 @@ void main() { return null; }); _addCallbackForWebDriver('get_updates', (arg) async { + listenForUpdates(); return updates.toJS; }); - _addCallbackForWebDriver('open', _open); + _addCallbackForWebDriver('open', (arg) => _open(arg, false)); + _addCallbackForWebDriver('open_only_vfs', (arg) => _open(arg, true)); _addCallbackForWebDriver('exec', _exec); _addCallbackForWebDriver('test_second', (arg) async { final endpoint = await database!.additionalConnection(); @@ -33,6 +36,28 @@ void main() { await second.dispose(); return true.toJS; }); + _addCallbackForWebDriver('assert_file', (arg) async { + final vfs = database!.fileSystem; + + final exists = await vfs.exists(FileType.database); + print('exists: $exists'); + if (exists != bool.parse(arg!)) { + return false.toJS; + } + + if (exists) { + // Try reading file contents + final buffer = await vfs.readFile(FileType.database); + return buffer.length.toJS; + } + + return true.toJS; + }); + _addCallbackForWebDriver('flush', (arg) async { + final vfs = database!.fileSystem; + await vfs.flush(); + return true.toJS; + }); document.getElementById('selfcheck')?.onClick.listen((event) async { print('starting'); @@ -88,7 +113,7 @@ Future _detectImplementations(String? _) async { }).toJS; } -Future _open(String? implementationName) async { +Future _open(String? implementationName, bool onlyOpenVfs) async { final sqlite = initializeSqlite(); Database db; var returnValue = implementationName; @@ -97,21 +122,33 @@ Future _open(String? implementationName) async { final split = implementationName.split(':'); db = await sqlite.connect(databaseName, StorageMode.values.byName(split[0]), - AccessMode.values.byName(split[1])); + AccessMode.values.byName(split[1]), + onlyOpenVfs: onlyOpenVfs); } else { - final result = await sqlite.connectToRecommended(databaseName); + final result = await sqlite.connectToRecommended(databaseName, + onlyOpenVfs: onlyOpenVfs); db = result.database; returnValue = '${result.storage.name}:${result.access.name}'; } + database = db; + // Make sure it works! - await db.select('SELECT database_host()'); + if (!onlyOpenVfs) { + await db.select('SELECT database_host()'); + listenForUpdates(); + } - db.updates.listen((_) => updates++); - database = db; return returnValue?.toJS; } +void listenForUpdates() { + if (!listeningForUpdates) { + listeningForUpdates = true; + database!.updates.listen((_) => updates++); + } +} + Future _exec(String? sql) async { await database!.execute(sql!); return null;