fix: use unique construct IDs for calculating aggregate analytics data (#3738)

lib/pangea/analytics_downloads/space_analytics_summary_model.dart

@@ -172,7 +172,7 @@ class SpaceAnalyticsSummaryModel {
       dataAvailable: model != null,
       level: model?.level,
       totalXP: model?.totalXP,
-      numLemmas: model?.vocabLemmas,
+      numLemmas: model?.numConstructs(ConstructTypeEnum.vocab),
       numLemmasUsedCorrectly: vocabLemmasCorrect?.over.length,
       numLemmasUsedIncorrectly: vocabLemmasCorrect?.under.length,
       numLemmasSmallXP:
@@ -180,7 +180,7 @@ class SpaceAnalyticsSummaryModel {
       numLemmasMediumXP:
           vocabLemmas?.thresholdedLemmas(start: 31, end: 200).length,
       numLemmasLargeXP: vocabLemmas?.thresholdedLemmas(start: 201).length,
-      numMorphConstructs: model?.grammarLemmas,
+      numMorphConstructs: model?.numConstructs(ConstructTypeEnum.morph),
       listMorphConstructs: morphLemmas?.lemmasToUses.entries
           .map((entry) => getCopy(entry.value.first))
           .toList(),

lib/pangea/analytics_misc/construct_list_model.dart

@@ -4,7 +4,6 @@ import 'package:flutter/material.dart';
 
 import 'package:collection/collection.dart';
 
-import 'package:fluffychat/pangea/analytics_misc/analytics_constants.dart';
 import 'package:fluffychat/pangea/analytics_misc/construct_type_enum.dart';
 import 'package:fluffychat/pangea/analytics_misc/construct_use_model.dart';
 import 'package:fluffychat/pangea/analytics_misc/constructs_model.dart';
@@ -28,39 +27,11 @@ class ConstructListModel {
   /// be accessed. It contains the same information as _constructMap, but sorted.
   List<ConstructUses> _constructList = [];
 
-  /// A list of unique vocab lemmas
-  List<String> _vocabLemmasList = [];
-
-  /// A list of unique grammar lemmas
-  List<String> _grammarLemmasList = [];
-
   /// [D] is the "compression factor". It determines how quickly
   /// or slowly the level grows relative to XP
 
   final double D = Environment.isStagingEnvironment ? 500 : 1500;
 
-  List<ConstructIdentifier> unlockedLemmas(
-    ConstructTypeEnum type, {
-    int threshold = 0,
-  }) {
-    final constructs = constructList(type: type);
-    final List<ConstructIdentifier> unlocked = [];
-    final constructsList =
-        type == ConstructTypeEnum.vocab ? _vocabLemmasList : _grammarLemmasList;
-
-    for (final lemma in constructsList) {
-      final matches = constructs.where((m) => m.lemma == lemma);
-      final totalPoints = matches.fold<int>(
-        0,
-        (total, match) => total + match.points,
-      );
-      if (totalPoints > threshold) {
-        unlocked.add(matches.first.id);
-      }
-    }
-    return unlocked;
-  }
-
   /// Analytics data consumed by widgets. Updated each time new analytics come in.
   int prevXP = 0;
   int totalXP = 0;
@@ -73,11 +44,6 @@ class ConstructListModel {
     updateConstructs(uses, offset);
   }
 
-  int get totalLemmas => _vocabLemmasList.length + _grammarLemmasList.length;
-  int get vocabLemmas => _vocabLemmasList.length;
-  int get grammarLemmas => _grammarLemmasList.length;
-  List<String> get lemmasList => _vocabLemmasList + _grammarLemmasList;
-
   /// Given a list of new construct uses, update the map of construct
   /// IDs to ConstructUses and re-sort the list of ConstructUses
   void updateConstructs(List<OneConstructUse> newUses, int offset) {
@@ -156,16 +122,6 @@ class ConstructListModel {
   }
 
   void _updateMetrics(int offset) {
-    _vocabLemmasList = constructList(type: ConstructTypeEnum.vocab)
-        .map((e) => e.lemma)
-        .toSet()
-        .toList();
-
-    _grammarLemmasList = constructList(type: ConstructTypeEnum.morph)
-        .map((e) => e.lemma)
-        .toSet()
-        .toList();
-
     prevXP = totalXP;
     totalXP = (_constructList.fold<int>(
           0,
@@ -179,6 +135,45 @@ class ConstructListModel {
     level = calculateLevelWithXp(totalXP);
   }
 
+  List<ConstructUses> constructList({ConstructTypeEnum? type}) => _constructList
+      .where(
+        (constructUse) => type == null || constructUse.constructType == type,
+      )
+      .toList();
+
+  // TODO; make this non-nullable, returning empty if not found
+  ConstructUses? getConstructUses(ConstructIdentifier identifier) {
+    final partialKey = "${identifier.lemma}-${identifier.type.string}";
+
+    if (_constructMap.containsKey(identifier.string)) {
+      // try to get construct use entry with full ID key
+      return _constructMap[identifier.string];
+    } else if (identifier.category == "other") {
+      // if the category passed to this function is "other", return the first
+      // construct use entry that starts with the partial key
+      return _constructMap.entries
+          .firstWhereOrNull((entry) => entry.key.startsWith(partialKey))
+          ?.value;
+    } else {
+      // if the category passed to this function is not "other", return the first
+      // construct use entry that starts with the partial key and ends with "other"
+      return _constructMap.entries
+          .firstWhereOrNull(
+            (entry) =>
+                entry.key.startsWith(partialKey) && entry.key.endsWith("other"),
+          )
+          ?.value;
+    }
+  }
+
+  List<ConstructUses> getConstructUsesByLemma(String lemma) => _constructList
+      .where(
+        (constructUse) => constructUse.lemma == lemma,
+      )
+      .toList();
+
+  int numConstructs(ConstructTypeEnum type) => constructList(type: type).length;
+
   int calculateLevelWithXp(int totalXP) {
     final doubleScore = (1 + sqrt((1 + (8.0 * totalXP / D)) / 2.0));
     if (!doubleScore.isNaN && doubleScore.isFinite) {
@@ -214,63 +209,30 @@ class ConstructListModel {
     return (xp < 0) ? 0 : xp;
   }
 
-  // TODO; make this non-nullable, returning empty if not found
-  ConstructUses? getConstructUses(ConstructIdentifier identifier) {
-    final partialKey = "${identifier.lemma}-${identifier.type.string}";
+  /// Unique construct identifiers with XP >= [threshold]
+  /// Used on analytics update to determine newly 'unlocked' constructs
+  List<ConstructIdentifier> unlockedLemmas(
+    ConstructTypeEnum type, {
+    int threshold = 0,
+  }) {
+    final constructs = constructList(type: type);
+    final List<ConstructIdentifier> unlocked = [];
+    final constructsList = [];
+    // type == ConstructTypeEnum.vocab ? _vocabLemmasList : _grammarLemmasList;
 
-    if (_constructMap.containsKey(identifier.string)) {
-      // try to get construct use entry with full ID key
-      return _constructMap[identifier.string];
-    } else if (identifier.category == "other") {
-      // if the category passed to this function is "other", return the first
-      // construct use entry that starts with the partial key
-      return _constructMap.entries
-          .firstWhereOrNull((entry) => entry.key.startsWith(partialKey))
-          ?.value;
-    } else {
-      // if the category passed to this function is not "other", return the first
-      // construct use entry that starts with the partial key and ends with "other"
-      return _constructMap.entries
-          .firstWhereOrNull(
-            (entry) =>
-                entry.key.startsWith(partialKey) && entry.key.endsWith("other"),
-          )
-          ?.value;
+    for (final lemma in constructsList) {
+      final matches = constructs.where((m) => m.lemma == lemma);
+      final totalPoints = matches.fold<int>(
+        0,
+        (total, match) => total + match.points,
+      );
+      if (totalPoints > threshold) {
+        unlocked.add(matches.first.id);
+      }
     }
+    return unlocked;
   }
 
-  List<ConstructUses> getConstructUsesByLemma(String lemma) {
-    return _constructList.where((constructUse) {
-      return constructUse.lemma == lemma;
-    }).toList();
-  }
-
-  List<ConstructUses> constructList({ConstructTypeEnum? type}) => _constructList
-      .where(
-        (constructUse) => type == null || constructUse.constructType == type,
-      )
-      .toList();
-
-  // uses where points < AnalyticConstants.xpForGreens
-  List<ConstructUses> get seeds => _constructList
-      .where(
-        (use) => use.points < AnalyticsConstants.xpForGreens,
-      )
-      .toList();
-
-  List<ConstructUses> get greens => _constructList
-      .where(
-        (use) =>
-            use.points >= AnalyticsConstants.xpForGreens &&
-            use.points < AnalyticsConstants.xpForFlower,
-      )
-      .toList();
-
-  List<ConstructUses> get flowers => _constructList
-      .where(
-        (use) => use.points >= AnalyticsConstants.xpForFlower,
-      )
-      .toList();
   // Not storing this for now to reduce memory load
   // It's only used by downloads, so doesn't need to be accessible on the fly
   Map<String, List<ConstructUses>> lemmasToUses({

lib/pangea/analytics_misc/get_analytics_controller.dart

@@ -572,9 +572,11 @@ class GetAnalyticsController extends BaseController {
     final response = await ConstructRepo.generateConstructSummary(request);
     final ConstructSummary summary = response.summary;
     summary.levelVocabConstructs = MatrixState
-        .pangeaController.getAnalytics.constructListModel.vocabLemmas;
+        .pangeaController.getAnalytics.constructListModel
+        .numConstructs(ConstructTypeEnum.vocab);
     summary.levelGrammarConstructs = MatrixState
-        .pangeaController.getAnalytics.constructListModel.grammarLemmas;
+        .pangeaController.getAnalytics.constructListModel
+        .numConstructs(ConstructTypeEnum.morph);
 
     final Room? analyticsRoom = await _client.getMyAnalyticsRoom(_l2!);
     if (analyticsRoom == null) {

lib/pangea/analytics_misc/level_up/level_up_manager.dart

@@ -3,6 +3,7 @@ import 'package:flutter/material.dart';
 import 'package:matrix/matrix.dart';
 
 import 'package:fluffychat/pangea/analytics_misc/client_analytics_extension.dart';
+import 'package:fluffychat/pangea/analytics_misc/construct_type_enum.dart';
 import 'package:fluffychat/pangea/constructs/construct_repo.dart';
 import 'package:fluffychat/pangea/events/constants/pangea_event_types.dart';
 import 'package:fluffychat/pangea/learning_settings/models/language_model.dart';
@@ -36,10 +37,10 @@ class LevelUpManager {
     //For on route change behavior, if added in the future
     shouldAutoPopup = true;
 
-    nextGrammar = MatrixState
-        .pangeaController.getAnalytics.constructListModel.grammarLemmas;
-    nextVocab = MatrixState
-        .pangeaController.getAnalytics.constructListModel.vocabLemmas;
+    nextGrammar = MatrixState.pangeaController.getAnalytics.constructListModel
+        .numConstructs(ConstructTypeEnum.morph);
+    nextVocab = MatrixState.pangeaController.getAnalytics.constructListModel
+        .numConstructs(ConstructTypeEnum.vocab);
 
     final LanguageModel? l2 =
         MatrixState.pangeaController.languageController.userL2;

lib/pangea/analytics_summary/learning_progress_indicators.dart

@@ -84,9 +84,9 @@ class LearningProgressIndicatorsState
   int uniqueLemmas(ProgressIndicatorEnum indicator) {
     switch (indicator) {
       case ProgressIndicatorEnum.morphsUsed:
-        return _constructsModel.grammarLemmas;
+        return _constructsModel.numConstructs(ConstructTypeEnum.morph);
       case ProgressIndicatorEnum.wordsUsed:
-        return _constructsModel.vocabLemmas;
+        return _constructsModel.numConstructs(ConstructTypeEnum.vocab);
       default:
         return 0;
     }