diff --git a/lib/src/backend/impls/objectbox/backend.dart b/lib/src/backend/impls/objectbox/backend.dart index 4619592b..2751c531 100644 --- a/lib/src/backend/impls/objectbox/backend.dart +++ b/lib/src/backend/impls/objectbox/backend.dart @@ -41,10 +41,11 @@ class _ObjectBoxBackendImpl implements ObjectBoxBackendInternal { late Completer _workerComplete; late StreamSubscription _workerHandler; - // `deleteOldestTile` tracking & debouncing - late int _dotLength; - late Timer _dotDebouncer; - late String? _dotStore; + // TODO: Verify if necessary and remove if not + //`removeOldestTilesAboveLimit` tracking & debouncing + //late int _rotalLength; + //late Timer _rotalDebouncer; + //late String? _rotalStore; Future?> _sendCmd({ required _WorkerCmdType type, @@ -93,8 +94,8 @@ class _ObjectBoxBackendImpl implements ObjectBoxBackendInternal { // Reset non-comms-related non-resource-intensive state _workerId = 0; _workerRes.clear(); - _dotStore = null; - _dotLength = 0; + //_rotalStore = null; + //_rotalLength = 0; // Prepare to recieve `SendPort` from worker _workerRes[0] = Completer(); @@ -157,7 +158,7 @@ class _ObjectBoxBackendImpl implements ObjectBoxBackendInternal { // Resource-intensive state cleanup only (other cleanup done during init) _sendPort = null; // Indicate ready for re-init await _workerHandler.cancel(); - _dotDebouncer.cancel(); + //_rotalDebouncer.cancel(); print('passed _workerHandler cancel'); @@ -281,13 +282,19 @@ class _ObjectBoxBackendImpl implements ObjectBoxBackendInternal { (await _sendCmd( type: _WorkerCmdType.deleteStore, args: {'storeName': storeName, 'url': url}, - ))!['wasOrphaned']; + ))!['wasOrphan']; @override - Future removeOldestTile({ + Future removeOldestTilesAboveLimit({ required String storeName, - required int numToRemove, - }) async { + required int tilesLimit, + }) async => + (await _sendCmd( + type: _WorkerCmdType.removeOldestTilesAboveLimit, + args: {'storeName': storeName, 'tilesLimit': tilesLimit}, + ))!['numOrphans']; + + /* FOR ABOVE METHOD // Attempts to avoid flooding worker with requests to delete oldest tile, // and 'batches' them instead @@ -315,13 +322,24 @@ class _ObjectBoxBackendImpl implements ObjectBoxBackendInternal { _dotDebouncer = Timer(const Duration(seconds: 1), () => _sendROTCmd(storeName)); _dotLength += numToRemove; - } - void _sendROTCmd(String storeName) { - _sendCmd( - type: _WorkerCmdType.removeOldestTile, - args: {'storeName': storeName, 'number': _dotLength}, - ); - _dotLength = 0; - } + // may need to be moved out + void _sendROTCmd(String storeName) { + _sendCmd( + type: _WorkerCmdType.removeOldestTile, + args: {'storeName': storeName, 'number': _dotLength}, + ); + _dotLength = 0; + } + */ + + @override + Future removeTilesOlderThan({ + required String storeName, + required DateTime expiry, + }) async => + (await _sendCmd( + type: _WorkerCmdType.removeTilesOlderThan, + args: {'storeName': storeName, 'expiry': expiry}, + ))!['numOrphans']; } diff --git a/lib/src/backend/impls/objectbox/worker.dart b/lib/src/backend/impls/objectbox/worker.dart index 5aee80c9..dd05f020 100644 --- a/lib/src/backend/impls/objectbox/worker.dart +++ b/lib/src/backend/impls/objectbox/worker.dart @@ -15,7 +15,8 @@ enum _WorkerCmdType { readTile, writeTile, deleteTile, - removeOldestTile, + removeOldestTilesAboveLimit, + removeTilesOlderThan, } Future _worker( @@ -27,10 +28,15 @@ Future _worker( int? maxReaders, }) input, ) async { + //! SETUP !// + // Setup comms final receivePort = ReceivePort(); - void sendRes(({int id, Map? data}) m) => - input.sendPort.send(m); + void sendRes({ + required int id, + Map? data, + }) => + input.sendPort.send((id: id, data: data)); // Initialise database final rootDirectory = await (input.rootDirectory == null @@ -46,13 +52,13 @@ Future _worker( // Respond with comms channel for future cmds sendRes( - ( - id: 0, - data: {'sendPort': receivePort.sendPort}, - ), + id: 0, + data: {'sendPort': receivePort.sendPort}, ); - // Setup util functions + //! UTIL FUNCTIONS !// + + /// Convert store name to database store object ObjectBoxStore? getStore(String storeName) { final query = root .box() @@ -63,7 +69,47 @@ Future _worker( return store; } - // Await cmds, perform work, and respond + /// Delete the specified tiles from the specified store + /// + /// Note that [tilesQuery] is not closed internally. Ensure it is closed after + /// usage. + /// + /// Returns whether each tile was actually deleted (whether it was an orphan), + /// in iteration order of [tilesQuery.find]. + Iterable deleteTiles({ + required String storeName, + required Query tilesQuery, + }) { + final stores = root.box(); + final tiles = root.box(); + + return tilesQuery.find().map((tile) { + // For the correct store, adjust the statistics + for (final store in tile.stores) { + if (store.name != storeName) continue; + stores.put( + store + ..numberOfTiles -= 1 + ..numberOfBytes -= tile.bytes.lengthInBytes, + mode: PutMode.update, + ); + break; + } + + // Remove the store relation from the tile + tile.stores.removeWhere((store) => store.name == storeName); + + // Delete the tile if it belongs to no stores + if (tile.stores.isEmpty) return tiles.remove(tile.id); + + // Otherwise just update the tile + tiles.put(tile, mode: PutMode.update); + return false; + }); + } + + //! MAIN LOOP !// + await for (final ({ int id, _WorkerCmdType type, @@ -78,17 +124,17 @@ Future _worker( if (cmd.args['deleteRoot'] == true) { rootDirectory.deleteSync(recursive: true); } + // TODO: Consider final message Isolate.exit(); case _WorkerCmdType.storeExists: sendRes( - ( - id: cmd.id, - data: { - 'exists': getStore(cmd.args['storeName']! as String) != null, - }, - ), + id: cmd.id, + data: { + 'exists': getStore(cmd.args['storeName']! as String) != null, + }, ); + break; case _WorkerCmdType.createStore: final storeName = cmd.args['storeName']! as String; @@ -107,7 +153,9 @@ Future _worker( } catch (e) { throw StoreAlreadyExists(storeName: storeName); } - sendRes((id: cmd.id, data: null)); + + sendRes(id: cmd.id); + break; case _WorkerCmdType.resetStore: final storeName = cmd.args['storeName']! as String; @@ -166,7 +214,9 @@ Future _worker( ); }, ); - sendRes((id: cmd.id, data: null)); + + sendRes(id: cmd.id); + break; case _WorkerCmdType.renameStore: final currentStoreName = cmd.args['currentStoreName']! as String; @@ -178,7 +228,8 @@ Future _worker( ..name = newStoreName, ); - sendRes((id: cmd.id, data: null)); + sendRes(id: cmd.id); + break; case _WorkerCmdType.deleteStore: root @@ -190,76 +241,74 @@ Future _worker( ..remove() ..close(); - sendRes((id: cmd.id, data: null)); + sendRes(id: cmd.id); + break; case _WorkerCmdType.getStoreSize: final storeName = cmd.args['storeName']! as String; sendRes( - ( - id: cmd.id, - data: { - 'size': (getStore(storeName) ?? - (throw StoreNotExists(storeName: storeName))) - .numberOfBytes / - 1024, - } - ), + id: cmd.id, + data: { + 'size': (getStore(storeName) ?? + (throw StoreNotExists(storeName: storeName))) + .numberOfBytes / + 1024, + }, ); + break; case _WorkerCmdType.getStoreLength: final storeName = cmd.args['storeName']! as String; sendRes( - ( - id: cmd.id, - data: { - 'length': (getStore(storeName) ?? - (throw StoreNotExists(storeName: storeName))) - .numberOfTiles, - } - ), + id: cmd.id, + data: { + 'length': (getStore(storeName) ?? + (throw StoreNotExists(storeName: storeName))) + .numberOfTiles, + }, ); + break; case _WorkerCmdType.getStoreHits: final storeName = cmd.args['storeName']! as String; sendRes( - ( - id: cmd.id, - data: { - 'hits': (getStore(storeName) ?? - (throw StoreNotExists(storeName: storeName))) - .hits, - } - ), + id: cmd.id, + data: { + 'hits': (getStore(storeName) ?? + (throw StoreNotExists(storeName: storeName))) + .hits, + }, ); + break; case _WorkerCmdType.getStoreMisses: final storeName = cmd.args['storeName']! as String; sendRes( - ( - id: cmd.id, - data: { - 'misses': (getStore(storeName) ?? - (throw StoreNotExists(storeName: storeName))) - .misses, - } - ), + id: cmd.id, + data: { + 'misses': (getStore(storeName) ?? + (throw StoreNotExists(storeName: storeName))) + .misses, + }, ); + break; case _WorkerCmdType.readTile: final query = root .box() .query(ObjectBoxTile_.url.equals(cmd.args['url']! as String)) .build(); - final tile = query.findUnique(); - query.close(); // TODO: Hits & misses - sendRes((id: cmd.id, data: {'tile': tile})); + sendRes(id: cmd.id, data: {'tile': query.findUnique()}); + + query.close(); + break; case _WorkerCmdType.writeTile: final storeName = cmd.args['storeName']! as String; @@ -290,7 +339,7 @@ Future _worker( tiles.put( ObjectBoxTile( url: url, - lastModified: DateTime.now(), + lastModified: DateTime.timestamp(), bytes: bytes!, )..stores.add(store), ); @@ -314,7 +363,7 @@ Future _worker( case (false, false): // Existing tile, update required tiles.put( existingTile! - ..lastModified = DateTime.now() + ..lastModified = DateTime.timestamp() ..bytes = bytes!, ); stores.putMany( @@ -335,103 +384,116 @@ Future _worker( }, ); - sendRes((id: cmd.id, data: null)); + sendRes(id: cmd.id); + break; case _WorkerCmdType.deleteTile: final storeName = cmd.args['storeName']! as String; final url = cmd.args['url']! as String; - final tiles = root.box(); - - // Find the tile by URL - final query = tiles.query(ObjectBoxTile_.url.equals(url)).build(); - final tile = query.findUnique(); - if (tile == null) { - sendRes((id: cmd.id, data: {'wasOrphaned': null})); - break; - } - - // For the correct store, adjust the statistics - for (final store in tile.stores) { - if (store.name != storeName) continue; - root.box().put( - store - ..numberOfTiles -= 1 - ..numberOfBytes -= tile.bytes.lengthInBytes, - mode: PutMode.update, - ); - break; - } - - // Remove the store relation from the tile - tile.stores.removeWhere((store) => store.name == storeName); + final query = root + .box() + .query(ObjectBoxTile_.url.equals(url)) + .build(); - // Delete the tile if it belongs to no stores - if (tile.stores.isEmpty) { - query - ..remove() - ..close(); - sendRes((id: cmd.id, data: {'wasOrphaned': true})); - break; - } + sendRes( + id: cmd.id, + data: { + 'wasOrphan': root + .runInTransaction( + TxMode.write, + () => deleteTiles(storeName: storeName, tilesQuery: query), + ) + .singleOrNull, + }, + ); - // Otherwise just update the tile query.close(); - tiles.put(tile, mode: PutMode.update); - sendRes((id: cmd.id, data: {'wasOrphaned': false})); + break; - case _WorkerCmdType.removeOldestTile: + case _WorkerCmdType.removeOldestTilesAboveLimit: final storeName = cmd.args['storeName']! as String; - final numTilesToRemove = cmd.args['number']! as int; - - final tiles = root.box(); + final tilesLimit = cmd.args['tilesLimit']! as int; - final tilesQuery = (tiles.query().order(ObjectBoxTile_.lastModified) + final tilesQuery = (root + .box() + .query() + .order(ObjectBoxTile_.lastModified) ..linkMany( ObjectBoxTile_.stores, ObjectBoxStore_.name.equals(storeName), )) .build(); - final deleteTiles = await tilesQuery - .stream() - .where((tile) => tile.stores.length == 1) - .take(numTilesToRemove) - .toList(); - tilesQuery.close(); - - if (deleteTiles.isEmpty) { - sendRes((id: cmd.id, data: null)); - break; - } final storeQuery = root .box() .query(ObjectBoxStore_.name.equals(storeName)) .build(); - root.runInTransaction( - TxMode.write, - () { - final store = storeQuery.findUnique() ?? - (throw StoreNotExists(storeName: storeName)); - storeQuery.close(); + sendRes( + id: cmd.id, + data: { + 'numOrphans': root + .runInTransaction( + TxMode.write, + () { + final store = storeQuery.findUnique() ?? + (throw StoreNotExists(storeName: storeName)); + + final numToRemove = store.numberOfTiles - tilesLimit; + + return numToRemove <= 0 + ? const Iterable.empty() + : deleteTiles( + storeName: storeName, + tilesQuery: tilesQuery..limit = numToRemove, + ); + }, + ) + .where((e) => e) + .length, + }, + ); - root.box().put( - store - ..numberOfTiles -= numTilesToRemove - ..numberOfBytes -= - deleteTiles.map((e) => e.bytes.lengthInBytes).sum, - mode: PutMode.update, - ); - tiles.removeMany(deleteTiles.map((e) => e.id).toList()); + storeQuery.close(); + tilesQuery.close(); + + break; + case _WorkerCmdType.removeTilesOlderThan: + final storeName = cmd.args['storeName']! as String; + final expiry = cmd.args['expiry']! as DateTime; + + final tilesQuery = (root.box().query( + ObjectBoxTile_.lastModified + .greaterThan(expiry.millisecondsSinceEpoch), + )..linkMany( + ObjectBoxTile_.stores, + ObjectBoxStore_.name.equals(storeName), + )) + .build(); + + sendRes( + id: cmd.id, + data: { + 'numOrphans': root + .runInTransaction( + TxMode.write, + () => deleteTiles( + storeName: storeName, + tilesQuery: tilesQuery, + ), + ) + .where((e) => e) + .length, }, ); - sendRes((id: cmd.id, data: null)); + tilesQuery.close(); + break; } } catch (e) { - sendRes((id: cmd.id, data: {'error': e})); + sendRes(id: cmd.id, data: {'error': e}); } } } diff --git a/lib/src/backend/interfaces/backend.dart b/lib/src/backend/interfaces/backend.dart index 94ca7107..6b8039f2 100644 --- a/lib/src/backend/interfaces/backend.dart +++ b/lib/src/backend/interfaces/backend.dart @@ -34,8 +34,12 @@ abstract interface class FMTCBackend { /// An abstract interface that FMTC will use to communicate with a storage /// 'backend' (usually one root) /// +/// Methods with a doc template in the doc string are for 'direct' public +/// invocation. +/// /// See [FMTCBackend] for more information. abstract interface class FMTCBackendInternal { + /// Generic description/name of this backend abstract final String friendlyIdentifier; /// {@template fmtc.backend.initialise} @@ -112,7 +116,9 @@ abstract interface class FMTCBackendInternal { required String storeName, }); - /// Change the name of the store named [currentStoreName] to [newStoreName] + /// {@template fmtc.backend.renameStore} + /// Change the name of the specified store to the specified new store name + /// {@endtemplate} Future renameStore({ required String currentStoreName, required String newStoreName, @@ -156,7 +162,8 @@ abstract interface class FMTCBackendInternal { required String url, }); - /// Create or update a tile + /// Create or update a tile (given a [url] and its [bytes]) in the specified + /// store /// /// If the tile already existed, it will be added to the specified store. /// Otherwise, [bytes] must be specified, and the tile will be created and @@ -170,23 +177,45 @@ abstract interface class FMTCBackendInternal { required Uint8List? bytes, }); - /// Remove the tile from the store, deleting it if orphaned + /// Remove the tile from the specified store, deleting it if was orphaned + /// + /// As tiles can belong to multiple stores, a tile cannot be safely 'truly' + /// deleted unless it does not belong to any other stores (it was an orphan). + /// A tile that is not an orphan will just be 'removed' from the specified + /// store. /// /// Returns: /// * `null` : if there was no existing tile /// * `true` : if the tile itself could be deleted (it was orphaned) - /// * `false`: if the tile still belonged to at least store + /// * `false`: if the tile still belonged to at least one other store Future deleteTile({ required String storeName, required String url, }); - /// {@template fmtc.backend.removeOldestTile} - /// Remove the specified number of tiles from the specified store, in the order - /// of oldest first, and where each tile does not belong to any other store + // TODO: Verify below and add to belower doc string + // + // It is recommended to invoke this operation as few times as possible, for + // example by debouncing, as this operation may be expensive. + + /// Remove tiles in excess of the specified limit from the specified store, + /// oldest first + /// + /// Returns the number of tiles that were actually deleted (they were + /// orphaned). See [deleteTile] for more information about orphan tiles. + Future removeOldestTilesAboveLimit({ + required String storeName, + required int tilesLimit, + }); + + /// {@template fmtc.backend.removeTilesOlderThan} + /// Remove tiles that were last modified after expiry from the specified store + /// + /// Returns the number of tiles that were actually deleted (they were + /// orphaned). See [deleteTile] for more information about orphan tiles. /// {@endtemplate} - Future removeOldestTile({ + Future removeTilesOlderThan({ required String storeName, - required int numToRemove, + required DateTime expiry, }); } diff --git a/lib/src/providers/image_provider.dart b/lib/src/providers/image_provider.dart index 757f1192..15746d04 100644 --- a/lib/src/providers/image_provider.dart +++ b/lib/src/providers/image_provider.dart @@ -105,7 +105,7 @@ class FMTCImageProvider extends ImageProvider { final needsUpdating = !needsCreating && (provider.settings.behavior == CacheBehavior.onlineFirst || (provider.settings.cachedValidDuration != Duration.zero && - DateTime.now().millisecondsSinceEpoch - + DateTime.timestamp().millisecondsSinceEpoch - existingTile.lastModified.millisecondsSinceEpoch > provider.settings.cachedValidDuration.inMilliseconds)); diff --git a/lib/src/store/manage.dart b/lib/src/store/manage.dart index b16c03fd..2109d7af 100644 --- a/lib/src/store/manage.dart +++ b/lib/src/store/manage.dart @@ -25,7 +25,7 @@ final class StoreManagement extends _WithBackendAccess { /// {@macro fmtc.backend.resetStore} Future reset() => _backend.resetStore(storeName: _storeName); - /// Rename the store to [newStoreName] + /// {@macro fmtc.backend.renameStore} /// /// The old [StoreDirectory] will still retain it's link to the old store, so /// always use the new returned value instead: returns a new [StoreDirectory] @@ -39,11 +39,9 @@ final class StoreManagement extends _WithBackendAccess { return StoreDirectory._(newStoreName); } - /// Delete all tiles older that were last modified before [expiry] - /// - /// Ignores [FMTCTileProviderSettings.cachedValidDuration]. - Future pruneTilesOlderThan({required DateTime expiry}) => - _backend.pruneTilesOlderThan(expiry: expiry); + /// {@macro fmtc.backend.removeTilesOlderThan} + Future removeTilesOlderThan({required DateTime expiry}) => + _backend.removeTilesOlderThan(storeName: _storeName, expiry: expiry); /// Retrieves the most recently modified tile from the store, extracts it's /// bytes, and renders them to an [Image] @@ -177,25 +175,3 @@ final class StoreManagement extends _WithBackendAccess { ); } } - -Future _pruneTilesOlderThanWorker(List args) async { - final db = Isar.openSync( - [DbStoreDescriptorSchema, DbTileSchema, DbMetadataSchema], - name: DatabaseTools.hash(args[0]).toString(), - directory: args[1], - inspector: false, - ); - - db.writeTxnSync( - () => db.tiles.deleteAllSync( - db.tiles - .where() - .lastModifiedLessThan(args[2]) - .findAllSync() - .map((t) => t.id) - .toList(), - ), - ); - - await db.close(); -}