fix categorization issues, combine broad/specific categories at the first stage of processing instead of during category sorting (#1136)

lib/pangea/models/analytics/construct_list_model.dart

@@ -79,6 +79,28 @@ class ConstructListModel {
       currentUses.setLastUsed(use.timeStamp);
       _constructMap[use.identifier.string] = currentUses;
     }
+
+    final broadKeys = _constructMap.keys.where((key) => key.endsWith('other'));
+    final replacedKeys = [];
+    for (final broadKey in broadKeys) {
+      final specificKeyPrefix = broadKey.split("-").first;
+      final specificKey = _constructMap.keys.firstWhereOrNull(
+        (key) =>
+            key != broadKey &&
+            key.startsWith(specificKeyPrefix) &&
+            !key.endsWith('other'),
+      );
+      if (specificKey == null) continue;
+      final broadConstructEntry = _constructMap[broadKey];
+      final specificConstructEntry = _constructMap[specificKey];
+      specificConstructEntry!.uses.addAll(broadConstructEntry!.uses);
+      _constructMap[specificKey] = specificConstructEntry;
+      replacedKeys.add(broadKey);
+    }
+
+    for (final key in replacedKeys) {
+      _constructMap.remove(key);
+    }
   }
 
   /// A list of ConstructUses, each of which contains a lemma and
@@ -91,57 +113,11 @@ class ConstructListModel {
 
   void _updateCategoriesToUses() {
     _categoriesToUses = {};
-
-    final Map<String, List<ConstructUses>> groupedMap = {};
-    for (final use in constructList()) {
-      // Step 1: Create a key based on type, lemma, and category
-      String key = use.id.string;
-
-      // If category is "other", find a more specific group if it exists
-      if (use.category.toLowerCase() == 'other') {
-        final String specificKeyPrefix = use.id.partialKey;
-        final String existingSpecificKey = groupedMap.keys.firstWhere(
-          (k) => k.startsWith(specificKeyPrefix) && !k.endsWith('other'),
-          orElse: () => '',
-        );
-
-        if (existingSpecificKey.isNotEmpty) {
-          key = existingSpecificKey;
-        }
-      }
-
-      // Add the object to the grouped map
-      groupedMap.putIfAbsent(key, () => []).add(use);
+    for (final ConstructUses use in constructList()) {
+      final category = use.category;
+      _categoriesToUses.putIfAbsent(category, () => []);
+      _categoriesToUses[category]!.add(use);
     }
-
-    // Step 2: Reorganize by category only
-    final Map<String, List<ConstructUses>> groupedByCategory = {};
-    for (final entry in groupedMap.entries) {
-      // Extract the category part from the key (assuming it's at the end)
-      final category = entry.key.split('-').last;
-
-      // Add each item in this entry to the groupedByCategory map under the single category key
-      groupedByCategory.putIfAbsent(category, () => []).addAll(entry.value);
-    }
-    final others = groupedByCategory.entries
-        .where((entry) => entry.key.toLowerCase() == 'other')
-        .toList();
-    if (others.length > 1) {
-      ErrorHandler.logError(
-        e: "More than one 'other' category in groupedByCategory",
-        data: {
-          "others": others.map((entry) {
-            List<String> useKeys =
-                entry.value.map((uses) => uses.id.string).toList();
-            if (useKeys.length > 10) {
-              useKeys = useKeys.sublist(0, 10);
-            }
-            ("${entry.key}: $useKeys");
-          }).toList(),
-        },
-      );
-    }
-    _categoriesToUses = groupedByCategory;
   }
 
   void _updateMetrics() {
@@ -191,7 +167,7 @@ class ConstructListModel {
     if (_constructMap.containsKey(identifier.string)) {
       // try to get construct use entry with full ID key
       return _constructMap[identifier.string];
-    } else if (identifier.category.toLowerCase() == "other") {
+    } else if (identifier.category == "other") {
       // if the category passed to this function is "other", return the first
       // construct use entry that starts with the partial key
       return _constructMap.entries
@@ -203,8 +179,7 @@ class ConstructListModel {
       return _constructMap.entries
           .firstWhereOrNull(
             (entry) =>
-                entry.key.startsWith(partialKey) &&
-                entry.key.toLowerCase().endsWith("other"),
+                entry.key.startsWith(partialKey) && entry.key.endsWith("other"),
           )
           ?.value;
     }

lib/pangea/models/analytics/construct_use_model.dart

@@ -39,7 +39,10 @@ class ConstructUses {
     _lastUsed = time;
   }
 
-  String get category => _category ?? "Other";
+  String get category {
+    if (_category == null || _category!.isEmpty) return "other";
+    return _category!.toLowerCase();
+  }
 
   ConstructIdentifier get id => ConstructIdentifier(
         lemma: lemma,

lib/pangea/models/analytics/constructs_model.dart

@@ -63,7 +63,7 @@ class OneConstructUse {
 
   /// For vocab constructs, this is the POS. For morph
   /// constructs, this is the morphological category.
-  String category;
+  String _category;
 
   ConstructTypeEnum constructType;
   ConstructUseTypeEnum useType;
@@ -79,10 +79,10 @@ class OneConstructUse {
     required this.lemma,
     required this.constructType,
     required this.metadata,
-    required this.category,
+    category,
     required this.form,
     this.id,
-  });
+  }) : _category = category ?? "other";
 
   String get chatId => metadata.roomId;
   String get msgId => metadata.eventId!;
@@ -122,6 +122,11 @@ class OneConstructUse {
         'id': id,
       };
 
+  String get category {
+    if (_category.isEmpty) return "other";
+    return _category.toLowerCase();
+  }
+
   static String getCategory(
     Map<String, dynamic> json,
     ConstructTypeEnum constructType,

lib/pangea/models/practice_activities.dart/practice_activity_model.dart

@@ -13,13 +13,13 @@ import 'package:sentry_flutter/sentry_flutter.dart';
 class ConstructIdentifier {
   final String lemma;
   final ConstructTypeEnum type;
-  final String category;
+  final String _category;
 
   ConstructIdentifier({
     required this.lemma,
     required this.type,
-    required this.category,
-  });
+    category,
+  }) : _category = category;
 
   factory ConstructIdentifier.fromJson(Map<String, dynamic> json) {
     final categoryEntry = json['cat'] ?? json['categories'];
@@ -55,6 +55,11 @@ class ConstructIdentifier {
     }
   }
 
+  String get category {
+    if (_category.isEmpty) return "other";
+    return _category.toLowerCase();
+  }
+
   Map<String, dynamic> toJson() {
     return {
       'lemma': lemma,
@@ -81,8 +86,9 @@ class ConstructIdentifier {
     return lemma.hashCode ^ type.hashCode;
   }
 
-  String get string =>
-      "$lemma-${type.string}${category != "" ? "-$category" : "-other"}";
+  String get string {
+    return "$lemma:${type.string}-$category".toLowerCase();
+  }
 
   String get partialKey => "$lemma-${type.string}";
 }

lib/pangea/widgets/chat_list/analytics_summary/analytics_popup/analytics_popup.dart

@@ -34,8 +34,8 @@ class AnalyticsPopupState extends State<AnalyticsPopup> {
     // Sort the list with custom logic
     entries.sort((a, b) {
       // Check if one of the keys is 'Other'
-      if (a.key == 'Other') return 1;
-      if (b.key == 'Other') return -1;
+      if (a.key.toLowerCase() == 'other') return 1;
+      if (b.key.toLowerCase() == 'other') return -1;
 
       // Sort by the length of the list in descending order
       final aTotalPoints = a.value.fold<int>(