From 77c4f711b04cc9c9865ba8bf61f9714791855d1c Mon Sep 17 00:00:00 2001 From: wcjord <32568597+wcjord@users.noreply.github.com> Date: Thu, 16 Jan 2025 10:47:31 -0500 Subject: [PATCH] =?UTF-8?q?feat(lemma=20meaning=20activity):=20widen=20dis?= =?UTF-8?q?tractor=20range,=20reduce=20lemmas=20w=E2=80=A6=20(#1469)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(lemma meaning activity): widen distractor range, reduce lemmas where meaning activity required * feat(lemma meaning activities): make distractor lemmas have same pos * dev(lemma meaning repo): use local storage instead of in-memory cache * fix(lemma meaning activity): explicitly prevent the same meanings in distractors * fix: dart formatting, deleted empty files --------- Co-authored-by: ggurdin <46800240+ggurdin@users.noreply.github.com> Co-authored-by: ggurdin --- .../analytics/repo/lemma_info_repo.dart | 59 ++----------- .../analytics/repo/lemma_info_request.dart | 4 + .../events/models/pangea_token_model.dart | 86 +++---------------- .../repo/lemma_activity_generator.dart | 74 +++++++++++++++- ... => lemma_meaning_activity_generator.dart} | 55 +++++++++++- lib/pangea/toolbar/repo/practice_repo.dart | 4 +- 6 files changed, 150 insertions(+), 132 deletions(-) rename lib/pangea/toolbar/repo/{word_meaning_activity_generator.dart => lemma_meaning_activity_generator.dart} (52%) diff --git a/lib/pangea/analytics/repo/lemma_info_repo.dart b/lib/pangea/analytics/repo/lemma_info_repo.dart index 1fb31901a..aa2adba33 100644 --- a/lib/pangea/analytics/repo/lemma_info_repo.dart +++ b/lib/pangea/analytics/repo/lemma_info_repo.dart @@ -3,6 +3,7 @@ import 'dart:developer'; import 'package:flutter/foundation.dart'; +import 'package:get_storage/get_storage.dart'; import 'package:http/http.dart'; import 'package:fluffychat/pangea/analytics/repo/lemma_info_request.dart'; @@ -15,27 +16,20 @@ import '../../common/config/environment.dart'; import '../../common/network/requests.dart'; class LemmaInfoRepo { - // In-memory cache with timestamps - static final Map _cache = {}; - static final Map _cacheTimestamps = {}; - - static const Duration _cacheDuration = Duration(days: 30); + static final GetStorage _lemmaStorage = GetStorage('lemma_storage'); static void set(LemmaInfoRequest request, LemmaInfoResponse response) { - _cache[request] = response; - - // set it to sometime in the future so we keep it in the cache for a while - _cacheTimestamps[request] = DateTime.now().add(const Duration(days: 365)); + _lemmaStorage.write(request.storageKey, response.toJson()); } static Future get( LemmaInfoRequest request, [ String? feedback, ]) async { - _clearExpiredEntries(); + final cachedJson = _lemmaStorage.read(request.storageKey); - if (_cache.containsKey(request)) { - final cached = _cache[request]!; + if (cachedJson != null) { + final cached = LemmaInfoResponse.fromJson(cachedJson); if (feedback == null) { // in this case, we just return the cached response @@ -57,7 +51,7 @@ class LemmaInfoRepo { data: request.toJson(), ); } else { - debugPrint('No cached response for lemma ${request.lemma}'); + debugPrint('No cached response for lemma ${request.lemma}, calling API'); } final Requests req = Requests( @@ -65,51 +59,16 @@ class LemmaInfoRepo { accessToken: MatrixState.pangeaController.userController.accessToken, ); - final requestBody = request.toJson(); final Response res = await req.post( url: PApiUrls.lemmaDictionary, - body: requestBody, + body: request.toJson(), ); final decodedBody = jsonDecode(utf8.decode(res.bodyBytes)); final response = LemmaInfoResponse.fromJson(decodedBody); - // Store the response and timestamp in the cache - _cache[request] = response; - _cacheTimestamps[request] = DateTime.now(); + set(request, response); return response; } - - /// From the cache, get a random set of cached definitions that are not for a specific lemma - static List getDistractorDefinitions( - String lemma, - int count, - ) { - _clearExpiredEntries(); - - final Set definitions = {}; - for (final entry in _cache.entries) { - if (entry.key.lemma != lemma) { - definitions.add(entry.value.meaning); - } - } - - definitions.toList().shuffle(); - - return definitions.take(count).toList(); - } - - static void _clearExpiredEntries() { - final now = DateTime.now(); - final expiredKeys = _cacheTimestamps.entries - .where((entry) => now.difference(entry.value) > _cacheDuration) - .map((entry) => entry.key) - .toList(); - - for (final key in expiredKeys) { - _cache.remove(key); - _cacheTimestamps.remove(key); - } - } } diff --git a/lib/pangea/analytics/repo/lemma_info_request.dart b/lib/pangea/analytics/repo/lemma_info_request.dart index 231732038..c9c0c04fa 100644 --- a/lib/pangea/analytics/repo/lemma_info_request.dart +++ b/lib/pangea/analytics/repo/lemma_info_request.dart @@ -40,4 +40,8 @@ class LemmaInfoRequest { @override int get hashCode => lemma.hashCode ^ partOfSpeech.hashCode ^ feedback.hashCode; + + String get storageKey { + return 'l:$lemma,p:$partOfSpeech,lang:$lemmaLang,l1:$userL1'; + } } diff --git a/lib/pangea/events/models/pangea_token_model.dart b/lib/pangea/events/models/pangea_token_model.dart index 9f833ebb8..76c96c9b3 100644 --- a/lib/pangea/events/models/pangea_token_model.dart +++ b/lib/pangea/events/models/pangea_token_model.dart @@ -1,5 +1,4 @@ import 'dart:developer'; -import 'dart:math'; import 'package:flutter/foundation.dart'; @@ -19,6 +18,8 @@ import 'package:fluffychat/pangea/events/models/pangea_token_text_model.dart'; import 'package:fluffychat/pangea/learning_settings/constants/language_constants.dart'; import 'package:fluffychat/pangea/toolbar/enums/activity_type_enum.dart'; import 'package:fluffychat/pangea/toolbar/models/practice_activity_model.dart'; +import 'package:fluffychat/pangea/toolbar/repo/lemma_activity_generator.dart'; +import 'package:fluffychat/pangea/toolbar/repo/lemma_meaning_activity_generator.dart'; import 'package:fluffychat/widgets/matrix.dart'; import '../../analytics/models/lemma.dart'; import '../../common/constants/model_keys.dart'; @@ -314,14 +315,14 @@ class PangeaToken { ]) { switch (a) { case ActivityTypeEnum.wordMeaning: - if (daysSinceLastUseByType(ActivityTypeEnum.wordMeaning) < 1) { + if (daysSinceLastUseByType(ActivityTypeEnum.wordMeaning) < 7) { return false; } if (isContentWord) { - return vocabConstruct.points < 30; + return vocabConstruct.points < 3; } else if (canBeDefined) { - return vocabConstruct.points < 5; + return vocabConstruct.points < 1; } else { return false; } @@ -398,10 +399,10 @@ class PangeaToken { ); return distractors.isNotEmpty; case ActivityTypeEnum.wordMeaning: - return LemmaInfoRepo.getDistractorDefinitions( + return LemmaMeaningActivityGenerator.canGenerateDistractors( lemma.text, - 1, - ).isNotEmpty; + pos, + ); case ActivityTypeEnum.emoji: case ActivityTypeEnum.wordFocusListening: case ActivityTypeEnum.hiddenWordListening: @@ -410,7 +411,8 @@ class PangeaToken { } Future canGenerateLemmaDistractors() async { - final distractors = await lemmaActivityDistractors(this); + final distractors = + await LemmaActivityGenerator().lemmaActivityDistractors(this); return distractors.isNotEmpty; } @@ -657,72 +659,4 @@ class PangeaToken { possibleDistractors.shuffle(); return possibleDistractors.take(3).toList(); } - - Future> lemmaActivityDistractors(PangeaToken token) async { - final List lemmas = MatrixState - .pangeaController.getAnalytics.constructListModel - .constructList(type: ConstructTypeEnum.vocab) - .map((c) => c.lemma) - .toSet() - .toList(); - - // Offload computation to an isolate - final Map distances = - await compute(_computeDistancesInIsolate, { - 'lemmas': lemmas, - 'target': token.lemma.text, - }); - - // Sort lemmas by distance - final sortedLemmas = distances.keys.toList() - ..sort((a, b) => distances[a]!.compareTo(distances[b]!)); - - // Take the shortest 4 - final choices = sortedLemmas.take(4).toList(); - if (!choices.contains(token.lemma.text)) { - final random = Random(); - choices[random.nextInt(4)] = token.lemma.text; - } - return choices; - } - - // isolate helper function - Map _computeDistancesInIsolate(Map params) { - final List lemmas = params['lemmas']; - final String target = params['target']; - - // Calculate Levenshtein distances - final Map distances = {}; - for (final lemma in lemmas) { - distances[lemma] = levenshteinDistanceSync(target, lemma); - } - return distances; - } - - int levenshteinDistanceSync(String s, String t) { - final int m = s.length; - final int n = t.length; - final List> dp = List.generate( - m + 1, - (_) => List.generate(n + 1, (_) => 0), - ); - - for (int i = 0; i <= m; i++) { - for (int j = 0; j <= n; j++) { - if (i == 0) { - dp[i][j] = j; - } else if (j == 0) { - dp[i][j] = i; - } else if (s[i - 1] == t[j - 1]) { - dp[i][j] = dp[i - 1][j - 1]; - } else { - dp[i][j] = 1 + - [dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1]] - .reduce((a, b) => a < b ? a : b); - } - } - } - - return dp[m][n]; - } } diff --git a/lib/pangea/toolbar/repo/lemma_activity_generator.dart b/lib/pangea/toolbar/repo/lemma_activity_generator.dart index 945e41355..1d13713f8 100644 --- a/lib/pangea/toolbar/repo/lemma_activity_generator.dart +++ b/lib/pangea/toolbar/repo/lemma_activity_generator.dart @@ -1,14 +1,18 @@ import 'dart:developer'; +import 'dart:math'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; import 'package:flutter_gen/gen_l10n/l10n.dart'; +import 'package:fluffychat/pangea/analytics/enums/construct_type_enum.dart'; +import 'package:fluffychat/pangea/events/models/pangea_token_model.dart'; import 'package:fluffychat/pangea/toolbar/enums/activity_type_enum.dart'; import 'package:fluffychat/pangea/toolbar/models/message_activity_request.dart'; import 'package:fluffychat/pangea/toolbar/models/multiple_choice_activity_model.dart'; import 'package:fluffychat/pangea/toolbar/models/practice_activity_model.dart'; +import 'package:fluffychat/widgets/matrix.dart'; class LemmaActivityGenerator { Future get( @@ -18,7 +22,7 @@ class LemmaActivityGenerator { debugger(when: kDebugMode && req.targetTokens.length != 1); final token = req.targetTokens.first; - final List choices = await token.lemmaActivityDistractors(token); + final List choices = await lemmaActivityDistractors(token); // TODO - modify MultipleChoiceActivity flow to allow no correct answer return MessageActivityResponse( @@ -36,4 +40,72 @@ class LemmaActivityGenerator { ), ); } + + Future> lemmaActivityDistractors(PangeaToken token) async { + final List lemmas = MatrixState + .pangeaController.getAnalytics.constructListModel + .constructList(type: ConstructTypeEnum.vocab) + .map((c) => c.lemma) + .toSet() + .toList(); + + // Offload computation to an isolate + final Map distances = + await compute(_computeDistancesInIsolate, { + 'lemmas': lemmas, + 'target': token.lemma.text, + }); + + // Sort lemmas by distance + final sortedLemmas = distances.keys.toList() + ..sort((a, b) => distances[a]!.compareTo(distances[b]!)); + + // Take the shortest 4 + final choices = sortedLemmas.take(4).toList(); + if (!choices.contains(token.lemma.text)) { + final random = Random(); + choices[random.nextInt(4)] = token.lemma.text; + } + return choices; + } + + // isolate helper function + Map _computeDistancesInIsolate(Map params) { + final List lemmas = params['lemmas']; + final String target = params['target']; + + // Calculate Levenshtein distances + final Map distances = {}; + for (final lemma in lemmas) { + distances[lemma] = levenshteinDistanceSync(target, lemma); + } + return distances; + } + + int levenshteinDistanceSync(String s, String t) { + final int m = s.length; + final int n = t.length; + final List> dp = List.generate( + m + 1, + (_) => List.generate(n + 1, (_) => 0), + ); + + for (int i = 0; i <= m; i++) { + for (int j = 0; j <= n; j++) { + if (i == 0) { + dp[i][j] = j; + } else if (j == 0) { + dp[i][j] = i; + } else if (s[i - 1] == t[j - 1]) { + dp[i][j] = dp[i - 1][j - 1]; + } else { + dp[i][j] = 1 + + [dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1]] + .reduce((a, b) => a < b ? a : b); + } + } + } + + return dp[m][n]; + } } diff --git a/lib/pangea/toolbar/repo/word_meaning_activity_generator.dart b/lib/pangea/toolbar/repo/lemma_meaning_activity_generator.dart similarity index 52% rename from lib/pangea/toolbar/repo/word_meaning_activity_generator.dart rename to lib/pangea/toolbar/repo/lemma_meaning_activity_generator.dart index 26935f261..366c52013 100644 --- a/lib/pangea/toolbar/repo/word_meaning_activity_generator.dart +++ b/lib/pangea/toolbar/repo/lemma_meaning_activity_generator.dart @@ -3,14 +3,17 @@ import 'package:flutter/material.dart'; import 'package:flutter_gen/gen_l10n/l10n.dart'; import 'package:fluffychat/pangea/analytics/enums/construct_type_enum.dart'; +import 'package:fluffychat/pangea/analytics/models/constructs_model.dart'; import 'package:fluffychat/pangea/analytics/repo/lemma_info_repo.dart'; import 'package:fluffychat/pangea/analytics/repo/lemma_info_request.dart'; +import 'package:fluffychat/pangea/analytics/repo/lemma_info_response.dart'; import 'package:fluffychat/pangea/toolbar/enums/activity_type_enum.dart'; import 'package:fluffychat/pangea/toolbar/models/message_activity_request.dart'; import 'package:fluffychat/pangea/toolbar/models/multiple_choice_activity_model.dart'; import 'package:fluffychat/pangea/toolbar/models/practice_activity_model.dart'; +import 'package:fluffychat/widgets/matrix.dart'; -class WordMeaningActivityGenerator { +class LemmaMeaningActivityGenerator { Future get( MessageActivityRequest req, BuildContext context, @@ -33,8 +36,7 @@ class WordMeaningActivityGenerator { final res = await LemmaInfoRepo.get(lemmaDefReq); - final choices = - LemmaInfoRepo.getDistractorDefinitions(lemmaDefReq.lemma, 3); + final choices = await getDistractorMeanings(lemmaDefReq, 3); if (!choices.contains(res.meaning)) { choices.add(res.meaning); @@ -57,4 +59,51 @@ class WordMeaningActivityGenerator { ), ); } + + static List eligibleDistractors(String lemma, String pos) { + return MatrixState.pangeaController.getAnalytics.constructListModel.uses + .where( + (c) => + c.lemma.toLowerCase() != lemma.toLowerCase() && + c.category.toLowerCase() == pos.toLowerCase() && + c.constructType == ConstructTypeEnum.vocab, + ) + .toList(); + } + + /// From the cache, get a random set of cached definitions that are not for a specific lemma + static Future> getDistractorMeanings( + LemmaInfoRequest req, + int count, + ) async { + final eligible = eligibleDistractors(req.lemma, req.partOfSpeech); + eligible.shuffle(); + + final List distractorConstructUses = + eligible.take(count).toList(); + + final List> futureDefs = []; + for (final construct in distractorConstructUses) { + futureDefs.add( + LemmaInfoRepo.get( + LemmaInfoRequest( + lemma: construct.lemma, + partOfSpeech: construct.category, + lemmaLang: req.lemmaLang, + userL1: req.userL1, + ), + ), + ); + } + + final Set distractorDefs = {}; + for (final def in await Future.wait(futureDefs)) { + distractorDefs.add(def.meaning); + } + + return distractorDefs.toList(); + } + + static bool canGenerateDistractors(String lemma, String pos) => + eligibleDistractors(lemma, pos).isNotEmpty; } diff --git a/lib/pangea/toolbar/repo/practice_repo.dart b/lib/pangea/toolbar/repo/practice_repo.dart index c6cf74edd..c02cbcf50 100644 --- a/lib/pangea/toolbar/repo/practice_repo.dart +++ b/lib/pangea/toolbar/repo/practice_repo.dart @@ -21,8 +21,8 @@ import 'package:fluffychat/pangea/toolbar/models/message_activity_request.dart'; import 'package:fluffychat/pangea/toolbar/models/practice_activity_model.dart'; import 'package:fluffychat/pangea/toolbar/repo/emoji_activity_generator.dart'; import 'package:fluffychat/pangea/toolbar/repo/lemma_activity_generator.dart'; +import 'package:fluffychat/pangea/toolbar/repo/lemma_meaning_activity_generator.dart'; import 'package:fluffychat/pangea/toolbar/repo/morph_activity_generator.dart'; -import 'package:fluffychat/pangea/toolbar/repo/word_meaning_activity_generator.dart'; import 'package:fluffychat/widgets/matrix.dart'; /// Represents an item in the completion cache. @@ -47,7 +47,7 @@ class PracticeGenerationController { final _morph = MorphActivityGenerator(); final _emoji = EmojiActivityGenerator(); final _lemma = LemmaActivityGenerator(); - final _wordMeaning = WordMeaningActivityGenerator(); + final _wordMeaning = LemmaMeaningActivityGenerator(); PracticeGenerationController() { _pangeaController = MatrixState.pangeaController;