// ignore_for_file: implementation_imports, depend_on_referenced_packages import 'dart:async'; import 'dart:math'; import 'package:collection/collection.dart'; import 'package:matrix/matrix.dart'; import 'package:sqflite_common/sqflite.dart'; import 'package:synchronized/synchronized.dart'; import 'package:fluffychat/pangea/analytics_data/derived_analytics_data_model.dart'; import 'package:fluffychat/pangea/analytics_misc/construct_type_enum.dart'; import 'package:fluffychat/pangea/analytics_misc/construct_use_model.dart'; import 'package:fluffychat/pangea/analytics_misc/construct_use_type_enum.dart'; import 'package:fluffychat/pangea/analytics_misc/constructs_event.dart'; import 'package:fluffychat/pangea/analytics_misc/constructs_model.dart'; import 'package:fluffychat/pangea/constructs/construct_identifier.dart'; import 'package:matrix/src/database/database_file_storage_stub.dart' if (dart.library.io) 'package:matrix/src/database/database_file_storage_io.dart'; import 'package:matrix/src/database/indexeddb_box.dart' if (dart.library.io) 'package:matrix/src/database/sqflite_box.dart'; class AnalyticsDatabase with DatabaseFileStorage { final String name; late BoxCollection _collection; late Box _lastEventTimestampBox; late Box _serverConstructsBox; late Box _localConstructsBox; late Box _aggregatedServerVocabConstructsBox; late Box _aggregatedLocalVocabConstructsBox; late Box _aggregatedServerMorphConstructsBox; late Box _aggregatedLocalMorphConstructsBox; late Box _derivedStatsBox; static const String _serverConstructsBoxName = 'box_server_constructs'; static const String _localConstructsBoxName = 'box_local_constructs'; /// Key is Tuple of construct lemma, type, and category static const String _aggregatedServerVocabConstructsBoxName = 'box_aggregated_server_vocab_constructs'; static const String _aggregatedLocalVocabConstructsBoxName = 'box_aggregated_local_vocab_constructs'; static const String _aggregatedServerMorphConstructsBoxName = 'box_aggregated_server_morph_constructs'; static const String _aggregatedLocalMorphConstructsBoxName = 'box_aggregated_local_morph_constructs'; static const String _derivedStatsBoxName = 'box_derived_stats'; static const String _lastEventTimestampBoxName = 'box_last_event_timestamp'; Database? database; /// Custom IdbFactory used to create the indexedDB. On IO platforms it would /// lead to an error to import "dart:indexed_db" so this is dynamically /// typed. final dynamic idbFactory; /// Custom SQFlite Database Factory used for high level operations on IO /// like delete. Set it if you want to use sqlite FFI. final DatabaseFactory? sqfliteFactory; static Future init( String name, { Database? database, dynamic idbFactory, DatabaseFactory? sqfliteFactory, Uri? fileStorageLocation, Duration? deleteFilesAfterDuration, }) async { final analyticsDatabase = AnalyticsDatabase._( name, database: database, idbFactory: idbFactory, sqfliteFactory: sqfliteFactory, fileStorageLocation: fileStorageLocation, deleteFilesAfterDuration: deleteFilesAfterDuration, ); await analyticsDatabase.open(); return analyticsDatabase; } AnalyticsDatabase._( this.name, { this.database, this.idbFactory, this.sqfliteFactory, Uri? fileStorageLocation, Duration? deleteFilesAfterDuration, }) { this.fileStorageLocation = fileStorageLocation; this.deleteFilesAfterDuration = deleteFilesAfterDuration; } final _lock = Lock(); Future open() async { _collection = await BoxCollection.open( name, { _lastEventTimestampBoxName, _serverConstructsBoxName, _localConstructsBoxName, _aggregatedServerVocabConstructsBoxName, _aggregatedLocalVocabConstructsBoxName, _aggregatedServerMorphConstructsBoxName, _aggregatedLocalMorphConstructsBoxName, _derivedStatsBoxName, }, sqfliteDatabase: database, sqfliteFactory: sqfliteFactory, idbFactory: idbFactory, version: MatrixSdkDatabase.version, ); _lastEventTimestampBox = _collection.openBox( _lastEventTimestampBoxName, ); _serverConstructsBox = _collection.openBox(_serverConstructsBoxName); _localConstructsBox = _collection.openBox(_localConstructsBoxName); _aggregatedServerVocabConstructsBox = _collection.openBox( _aggregatedServerVocabConstructsBoxName, ); _aggregatedLocalVocabConstructsBox = _collection.openBox( _aggregatedLocalVocabConstructsBoxName, ); _aggregatedServerMorphConstructsBox = _collection.openBox( _aggregatedServerMorphConstructsBoxName, ); _aggregatedLocalMorphConstructsBox = _collection.openBox( _aggregatedLocalMorphConstructsBoxName, ); _derivedStatsBox = _collection.openBox(_derivedStatsBoxName); } Future delete() async { await _collection.deleteDatabase( database?.path ?? name, sqfliteFactory ?? idbFactory, ); } Future clear() async { _lastEventTimestampBox.clearQuickAccessCache(); _serverConstructsBox.clearQuickAccessCache(); _localConstructsBox.clearQuickAccessCache(); _aggregatedServerVocabConstructsBox.clearQuickAccessCache(); _aggregatedLocalVocabConstructsBox.clearQuickAccessCache(); _aggregatedServerMorphConstructsBox.clearQuickAccessCache(); _aggregatedLocalMorphConstructsBox.clearQuickAccessCache(); _derivedStatsBox.clearQuickAccessCache(); await _collection.clear(); } Future _transaction(Future Function() action) { return _lock.synchronized(action); } Box _aggBox(ConstructTypeEnum type, bool local) => switch ((type, local)) { (ConstructTypeEnum.vocab, true) => _aggregatedLocalVocabConstructsBox, (ConstructTypeEnum.vocab, false) => _aggregatedServerVocabConstructsBox, (ConstructTypeEnum.morph, true) => _aggregatedLocalMorphConstructsBox, (ConstructTypeEnum.morph, false) => _aggregatedServerMorphConstructsBox, }; String _langKey(String key, String language) => '$language|$key'; bool _isLanguageKey(String key, String language) => key.startsWith('$language|'); Future getUserID() => _lastEventTimestampBox.get('user_id'); Future getLastUpdated(String language) async { final entry = await _lastEventTimestampBox.get( _langKey('last_updated', language), ); if (entry == null) return null; return DateTime.tryParse(entry); } Future getLastEventTimestamp(String language) async { final timestampString = await _lastEventTimestampBox.get( _langKey('last_event_timestamp', language), ); if (timestampString == null) return null; return DateTime.parse(timestampString); } Future getCurrentLanguage() async { return _lastEventTimestampBox.get('current_language'); } Future getDerivedStats(String language) async { final raw = await _derivedStatsBox.get(_langKey('derived_stats', language)); return raw == null ? DerivedAnalyticsDataModel() : DerivedAnalyticsDataModel.fromJson(Map.from(raw)); } Future> getUses( String language, { int? count, String? roomId, DateTime? since, List? types, }) async { final stopwatch = Stopwatch()..start(); final results = []; bool addUseIfValid(OneConstructUse use) { if (since != null && use.timeStamp.isBefore(since)) { return false; // stop iteration entirely } if (roomId != null && use.metadata.roomId != roomId) { return true; // skip but continue } if (types != null && !types.contains(use.useType)) { return true; // skip but continue } results.add(use); return count == null || results.length < count; } // ---- Local uses ---- final localUses = await getLocalUses(language) ..sort((a, b) => b.timeStamp.compareTo(a.timeStamp)); for (final use in localUses) { if (!addUseIfValid(use)) break; } if (count != null && results.length >= count) { stopwatch.stop(); Logs().i("Get uses took ${stopwatch.elapsedMilliseconds} ms"); return results; } // ---- Server uses ---- final serverKeys = (await _serverConstructsBox.getAllKeys()) .where((key) => _isLanguageKey(key, language)) // Filter out malformed or legacy keys that don't have a timestamp .where((key) { final parts = key.split('|'); return parts.length >= 3 && int.tryParse(parts[2]) != null; }) .sorted((a, b) { final aTimestamp = int.parse(a.split('|')[2]); final bTimestamp = int.parse(b.split('|')[2]); return bTimestamp.compareTo(aTimestamp); }); for (final key in serverKeys) { final serverUses = await getServerUses(key) ..sort((a, b) => b.timeStamp.compareTo(a.timeStamp)); for (final use in serverUses) { if (!addUseIfValid(use)) break; } if (count != null && results.length >= count) break; } stopwatch.stop(); Logs().i("Get uses took ${stopwatch.elapsedMilliseconds} ms"); return results; } Future> getLocalUses(String language) async { final List uses = []; final localKeys = (await _localConstructsBox.getAllKeys()) .where((key) => _isLanguageKey(key, language)) .toList(); final localValues = await _localConstructsBox.getAll(localKeys); for (final rawList in localValues) { if (rawList == null) continue; for (final raw in rawList) { final use = OneConstructUse.fromJson(Map.from(raw)); uses.add(use); } } return uses; } Future> getServerUses(String key) async { final List uses = []; final serverValues = await _serverConstructsBox.get(key); if (serverValues == null) return []; for (final entry in serverValues) { uses.add(OneConstructUse.fromJson(Map.from(entry))); } return uses; } Future getLocalConstructCount(String language) async { final keys = (await _localConstructsBox.getAllKeys()).where( (key) => _isLanguageKey(key, language), ); return keys.length; } Future getConstructUse( List ids, String language, ) async { assert(ids.isNotEmpty); final ConstructUses construct = ConstructUses( uses: [], constructType: ids.first.type, lemma: ids.first.lemma, category: ids.first.category, ); for (final id in ids) { final key = id.storageKey; ConstructUses? server; ConstructUses? local; final serverBox = _aggBox(id.type, false); final localBox = _aggBox(id.type, true); final serverRaw = await serverBox.get(_langKey(key, language)); if (serverRaw != null) { server = ConstructUses.fromJson(Map.from(serverRaw)); } final localRaw = await localBox.get(_langKey(key, language)); if (localRaw != null) { local = ConstructUses.fromJson(Map.from(localRaw)); } if (server != null) construct.merge(server); if (local != null) construct.merge(local); } return construct; } Future> getConstructUses( Map> ids, String language, ) async { final Map results = {}; for (final entry in ids.entries) { final construct = await getConstructUse(entry.value, language); results[entry.key] = construct; } return results; } Future clearLocalConstructData(String language) async { await _transaction(() async { final localKeys = (await _localConstructsBox.getAllKeys()) .where((key) => _isLanguageKey(key, language)) .toList(); final localVocabAggKeys = (await _aggregatedLocalVocabConstructsBox.getAllKeys()) .where((key) => _isLanguageKey(key, language)) .toList(); final localMorphAggKeys = (await _aggregatedLocalMorphConstructsBox.getAllKeys()) .where((key) => _isLanguageKey(key, language)) .toList(); await _localConstructsBox.deleteAll(localKeys); await _aggregatedLocalVocabConstructsBox.deleteAll(localVocabAggKeys); await _aggregatedLocalMorphConstructsBox.deleteAll(localMorphAggKeys); }); } /// Group uses by aggregate key Map> _groupUses( List uses, String language, ) { final Map> grouped = {}; for (final u in uses) { final key = _langKey(u.identifier.storageKey, language); (grouped[key] ??= []).add(u); } return grouped; } Map _aggregateConstructs( Map> groups, Map?> existingRaw, ) { final Map updates = {}; for (final entry in groups.entries) { final key = entry.key; final usesForKey = entry.value; final raw = existingRaw[key]; ConstructUses model; if (raw != null) { model = ConstructUses.fromJson(Map.from(raw)); } else { final u = usesForKey.first; model = ConstructUses( uses: [], constructType: u.constructType, lemma: u.lemma, category: u.category, ); } model.addUses(usesForKey); updates[key] = model; } return updates; } Future> _aggregateFromBox( Box box, Map> grouped, ) async { final keys = grouped.keys.toList(); final existing = await box.getAll(keys); final existingMap = Map.fromIterables(keys, existing); return _aggregateConstructs(grouped, existingMap); } Future> getAggregatedConstructs( ConstructTypeEnum type, String language, ) async { Map combined = {}; final stopwatch = Stopwatch()..start(); final localKeys = (await _aggBox( type, true, ).getAllKeys()).where((key) => _isLanguageKey(key, language)).toList(); final serverKeys = (await _aggBox( type, false, ).getAllKeys()).where((key) => _isLanguageKey(key, language)).toList(); final serverValues = await _aggBox(type, false).getAll(serverKeys); final serverConstructs = serverValues .map((e) => ConstructUses.fromJson(Map.from(e!))) .toList(); final serverAgg = Map.fromIterables(serverKeys, serverConstructs); if (localKeys.isEmpty) { combined = serverAgg; } else { final localValues = await _aggBox(type, true).getAll(localKeys); final localConstructs = localValues .map((e) => ConstructUses.fromJson(Map.from(e!))) .toList(); final localAgg = Map.fromIterables(localKeys, localConstructs); combined = Map.from(serverAgg); for (final entry in localAgg.entries) { final key = entry.key; final localModel = entry.value; if (combined.containsKey(key)) { final serverModel = combined[key]!; serverModel.merge(localModel); combined[key] = serverModel; } else { combined[key] = localModel; } } } stopwatch.stop(); Logs().i("Combining aggregates took ${stopwatch.elapsedMilliseconds} ms"); return combined.values.toList(); } Future updateUserID(String userID) { return _transaction(() async { await _lastEventTimestampBox.put('user_id', userID); }); } Future updateCurrentLanguage(String language) { return _transaction(() async { await _lastEventTimestampBox.put('current_language', language); }); } Future _updateLastUpdated(DateTime timestamp, String language) async { return _transaction(() async { await _lastEventTimestampBox.put( _langKey('last_updated', language), timestamp.toIso8601String(), ); }); } Future updateXPOffset(int offset, String language) async { return _transaction(() async { final stats = await getDerivedStats(language); final updatedStats = stats.copyWithOffset(offset); await _derivedStatsBox.put( _langKey('derived_stats', language), updatedStats.toJson(), ); }); } Future updateTotalXP(int totalXP, String language) { return _transaction(() async { final stats = await getDerivedStats(language); final updatedStats = stats.copyWithTotalXP(totalXP); await _derivedStatsBox.put( _langKey('derived_stats', language), updatedStats.toJson(), ); }); } Future updateDerivedStats( DerivedAnalyticsDataModel newStats, String language, ) => _derivedStatsBox.put( _langKey('derived_stats', language), newStats.toJson(), ); Future updateServerAnalytics( List events, String language, ) async { if (events.isEmpty) return; final stopwatch = Stopwatch()..start(); await _transaction(() async { final lastUpdated = await getLastEventTimestamp(language); DateTime mostRecent = lastUpdated ?? events.first.event.originServerTs; final existingKeys = (await _serverConstructsBox.getAllKeys()) .where((key) => _isLanguageKey(key, language)) .toSet(); final List aggregatedVocabUses = []; final List aggregatedMorphUses = []; final Map> pendingWrites = {}; for (final event in events) { final ts = event.event.originServerTs; final key = TupleKey( event.event.eventId, ts.millisecondsSinceEpoch.toString(), ).toString(); if (lastUpdated != null && ts.isBefore(lastUpdated)) continue; if (existingKeys.contains(_langKey(key, language))) continue; if (ts.isAfter(mostRecent)) mostRecent = ts; pendingWrites[key] = event.content.uses; for (final u in event.content.uses) { u.constructType == ConstructTypeEnum.vocab ? aggregatedVocabUses.add(u) : aggregatedMorphUses.add(u); } } if (pendingWrites.isEmpty) return; // Write events sequentially for (final e in pendingWrites.entries) { _serverConstructsBox.put( _langKey(e.key, language), e.value.map((u) => u.toJson()).toList(), ); } // Update aggregates final aggVocabUpdates = await _aggregateFromBox( _aggregatedServerVocabConstructsBox, _groupUses(aggregatedVocabUses, language), ); for (final entry in aggVocabUpdates.entries) { await _aggregatedServerVocabConstructsBox.put( entry.key, entry.value.toJson(), ); } final aggMorphUpdates = await _aggregateFromBox( _aggregatedServerMorphConstructsBox, _groupUses(aggregatedMorphUses, language), ); for (final entry in aggMorphUpdates.entries) { await _aggregatedServerMorphConstructsBox.put( entry.key, entry.value.toJson(), ); } // Update timestamp await _lastEventTimestampBox.put( _langKey('last_event_timestamp', language), mostRecent.toIso8601String(), ); }); await _updateLastUpdated(DateTime.now(), language); stopwatch.stop(); Logs().i( "Server analytics update took ${stopwatch.elapsedMilliseconds} ms", ); } Future updateLocalAnalytics( List uses, String language, ) async { if (uses.isEmpty) return; final stopwatch = Stopwatch()..start(); await _transaction(() async { // Store local constructs final key = DateTime.now().millisecondsSinceEpoch; _localConstructsBox.put( _langKey(key.toString(), language), uses.map((u) => u.toJson()).toList(), ); final List vocabUses = []; final List morphUses = []; for (final u in uses) { u.constructType == ConstructTypeEnum.vocab ? vocabUses.add(u) : morphUses.add(u); } // Update aggregates final aggVocabUpdates = await _aggregateFromBox( _aggregatedLocalVocabConstructsBox, _groupUses(vocabUses, language), ); for (final entry in aggVocabUpdates.entries) { await _aggregatedLocalVocabConstructsBox.put( entry.key, entry.value.toJson(), ); } final aggMorphUpdates = await _aggregateFromBox( _aggregatedLocalMorphConstructsBox, _groupUses(morphUses, language), ); for (final entry in aggMorphUpdates.entries) { await _aggregatedLocalMorphConstructsBox.put( entry.key, entry.value.toJson(), ); } }); await _updateLastUpdated(DateTime.now(), language); stopwatch.stop(); Logs().i("Local analytics update took ${stopwatch.elapsedMilliseconds} ms"); } }