simplify message token renderer (#4994)

* simplify message token renderer

* token rendering and new word collection for tokens in activity summary / menu

* make tokens hoverable

lib/pages/chat/events/html_message.dart

@@ -23,6 +23,7 @@ import 'package:fluffychat/pangea/toolbar/reading_assistance/tokens_util.dart';
 import 'package:fluffychat/utils/event_checkbox_extension.dart';
 import 'package:fluffychat/widgets/avatar.dart';
 import 'package:fluffychat/widgets/future_loading_dialog.dart';
+import 'package:fluffychat/widgets/hover_builder.dart';
 import 'package:fluffychat/widgets/matrix.dart';
 import 'package:fluffychat/widgets/mxc_image.dart';
 import '../../../utils/url_launcher.dart';
@@ -388,8 +389,6 @@ class HtmlMessage extends StatelessWidget {
 
     // #Pangea
     final renderer = TokenRenderingUtil(
-      pangeaMessageEvent: pangeaMessageEvent,
-      readingAssistanceMode: readingAssistanceMode,
       existingStyle: pangeaMessageEvent != null
           ? textStyle.merge(
               AppConfig.messageTextStyle(
@@ -398,14 +397,21 @@ class HtmlMessage extends StatelessWidget {
               ),
             )
           : textStyle,
-      overlayController: overlayController,
-      isTransitionAnimation: isTransitionAnimation,
     );
 
-    final fontSize = renderer.fontSize(context) ?? this.fontSize;
+    double fontSize = this.fontSize;
+    if (readingAssistanceMode == ReadingAssistanceMode.practiceMode) {
+      fontSize = (overlayController != null && overlayController!.maxWidth > 600
+              ? Theme.of(context).textTheme.titleLarge?.fontSize
+              : Theme.of(context).textTheme.bodyLarge?.fontSize) ??
+          this.fontSize;
+    }
+
+    final underlineColor = Theme.of(context).colorScheme.primary.withAlpha(200);
+
     final newTokens =
         pangeaMessageEvent != null && !pangeaMessageEvent!.ownMessage
-            ? TokensUtil.getNewTokens(pangeaMessageEvent!)
+            ? TokensUtil.getNewTokensByEvent(pangeaMessageEvent!)
             : [];
     // Pangea#
 
@@ -428,8 +434,9 @@ class HtmlMessage extends StatelessWidget {
 
         final isNew = token != null && newTokens.contains(token.text);
         final tokenWidth = renderer.tokenTextWidthForContainer(
-          context,
           node.text,
+          Theme.of(context).colorScheme.primary.withAlpha(200),
+          fontSize: fontSize,
         );
 
         return TextSpan(
@@ -451,22 +458,16 @@ class HtmlMessage extends StatelessWidget {
                           overlayController!.onClickOverlayMessageToken(token),
                       textColor: textColor,
                     ),
-                  if (renderer.showCenterStyling &&
+                  if (readingAssistanceMode ==
+                          ReadingAssistanceMode.practiceMode &&
                       token != null &&
                       overlayController != null)
                     TokenPracticeButton(
                       token: token,
                       controller: overlayController!.practiceController,
                       textStyle: renderer.style(
-                        context,
-                        color: renderer.backgroundColor(
-                          context,
-                          selected,
-                          highlighted,
-                          isNew,
-                          readingAssistanceMode ==
-                              ReadingAssistanceMode.practiceMode,
-                        ),
+                        fontSize: fontSize,
+                        underlineColor: underlineColor,
                       ),
                       width: tokenWidth,
                       textColor: textColor,
@@ -486,34 +487,39 @@ class HtmlMessage extends StatelessWidget {
                         onTap: onClick != null && token != null
                             ? () => onClick?.call(token)
                             : null,
-                        child: RichText(
-                          textDirection: pangeaMessageEvent?.textDirection,
-                          text: TextSpan(
-                            children: [
-                              LinkifySpan(
-                                text: node.text.trim(),
-                                style: renderer.style(
-                                  context,
-                                  color: renderer.backgroundColor(
-                                    context,
-                                    selected,
-                                    highlighted,
-                                    isNew,
-                                    readingAssistanceMode ==
-                                        ReadingAssistanceMode.practiceMode,
+                        child: HoverBuilder(
+                          builder: (context, hovered) {
+                            return RichText(
+                              textDirection: pangeaMessageEvent?.textDirection,
+                              text: TextSpan(
+                                children: [
+                                  LinkifySpan(
+                                    text: node.text.trim(),
+                                    style: renderer.style(
+                                      fontSize: fontSize,
+                                      underlineColor: underlineColor,
+                                      selected: selected,
+                                      highlighted: highlighted,
+                                      isNew: isNew,
+                                      practiceMode: readingAssistanceMode ==
+                                          ReadingAssistanceMode.practiceMode,
+                                      hovered: hovered,
+                                    ),
+                                    linkStyle: linkStyle,
+                                    onOpen: (url) =>
+                                        UrlLauncher(context, url.url)
+                                            .launchUrl(),
                                   ),
-                                ),
-                                linkStyle: linkStyle,
-                                onOpen: (url) =>
-                                    UrlLauncher(context, url.url).launchUrl(),
+                                ],
                               ),
-                            ],
-                          ),
+                            );
+                          },
                         ),
                       ),
                     ),
                   ),
-                  if (renderer.showCenterStyling &&
+                  if (readingAssistanceMode ==
+                          ReadingAssistanceMode.practiceMode &&
                       token != null &&
                       overlayController != null)
                     ListenableBuilder(
@@ -657,14 +663,8 @@ class HtmlMessage extends StatelessWidget {
                     TextSpan(
                       text: '• ',
                       style: renderer.style(
-                        context,
-                        color: renderer.backgroundColor(
-                          context,
-                          false,
-                          false,
-                          false,
-                          false,
-                        ),
+                        underlineColor: underlineColor,
+                        fontSize: fontSize,
                       ),
                     ),
                   // Pangea#
@@ -675,14 +675,8 @@ class HtmlMessage extends StatelessWidget {
                       // #Pangea
                       // style: textStyle,
                       style: renderer.style(
-                        context,
-                        color: renderer.backgroundColor(
-                          context,
-                          false,
-                          false,
-                          false,
-                          false,
-                        ),
+                        underlineColor: underlineColor,
+                        fontSize: fontSize,
                       ),
                       // Pangea#
                     ),

lib/pangea/activity_planner/activity_plan_model.dart

@@ -1,8 +1,12 @@
 import 'package:flutter/foundation.dart';
+import 'package:flutter/material.dart';
 
 import 'package:fluffychat/pangea/activity_planner/activity_plan_request.dart';
 import 'package:fluffychat/pangea/common/config/environment.dart';
 import 'package:fluffychat/pangea/common/constants/model_keys.dart';
+import 'package:fluffychat/pangea/events/models/pangea_token_model.dart';
+import 'package:fluffychat/pangea/events/models/pangea_token_text_model.dart';
+import 'package:fluffychat/pangea/lemmas/lemma.dart';
 
 class ActivityPlanModel {
   final String activityId;
@@ -182,6 +186,21 @@ class Vocab {
     );
   }
 
+  PangeaToken asToken() {
+    final text = PangeaTokenText(
+      content: lemma,
+      length: lemma.characters.length,
+      offset: 0,
+    );
+
+    return PangeaToken(
+      text: text,
+      lemma: Lemma(text: lemma, saveVocab: true, form: lemma),
+      pos: pos,
+      morph: {},
+    );
+  }
+
   Map<String, dynamic> toJson() {
     return {
       'lemma': lemma,

lib/pangea/activity_sessions/activity_session_chat/activity_vocab_widget.dart

@@ -6,7 +6,11 @@ import 'package:fluffychat/pangea/analytics_misc/construct_type_enum.dart';
 import 'package:fluffychat/pangea/common/utils/overlay.dart';
 import 'package:fluffychat/pangea/constructs/construct_identifier.dart';
 import 'package:fluffychat/pangea/events/models/pangea_token_text_model.dart';
+import 'package:fluffychat/pangea/toolbar/reading_assistance/token_rendering_util.dart';
+import 'package:fluffychat/pangea/toolbar/reading_assistance/tokens_util.dart';
+import 'package:fluffychat/pangea/toolbar/token_rendering_mixin.dart';
 import 'package:fluffychat/pangea/toolbar/word_card/word_zoom_widget.dart';
+import 'package:fluffychat/widgets/hover_builder.dart';
 import 'package:fluffychat/widgets/matrix.dart';
 
 class ActivityVocabWidget extends StatelessWidget {
@@ -46,7 +50,7 @@ class ActivityVocabWidget extends StatelessWidget {
   }
 }
 
-class _VocabChips extends StatelessWidget {
+class _VocabChips extends StatefulWidget {
   final List<Vocab> vocab;
   final String targetId;
   final String langCode;
@@ -59,8 +63,39 @@ class _VocabChips extends StatelessWidget {
     required this.usedVocab,
   });
 
-  void _onTap(Vocab v, BuildContext context) {
-    final target = "$targetId-${v.lemma}";
+  @override
+  State<_VocabChips> createState() => _VocabChipsState();
+}
+
+class _VocabChipsState extends State<_VocabChips> with TokenRenderingMixin {
+  Vocab? _selectedVocab;
+
+  @override
+  void dispose() {
+    TokensUtil.clearNewTokenCache();
+    super.dispose();
+  }
+
+  void _onTap(
+    Vocab v,
+    bool isNew,
+  ) {
+    setState(() {
+      _selectedVocab = v;
+    });
+
+    final target = "${widget.targetId}-${v.lemma}";
+    if (isNew) {
+      final token = v.asToken();
+      collectNewToken(
+        "activity_tokens",
+        widget.targetId,
+        token,
+        Matrix.of(context).analyticsDataService,
+      ).then((_) {
+        if (mounted) setState(() {});
+      });
+    }
     OverlayUtil.showPositionedCard(
       overlayKey: target,
       context: context,
@@ -75,9 +110,10 @@ class _VocabChips extends StatelessWidget {
           type: ConstructTypeEnum.vocab,
           category: v.pos,
         ),
-        langCode: langCode,
+        langCode: widget.langCode,
         onClose: () {
           MatrixState.pAnyState.closeOverlay(target);
+          setState(() => _selectedVocab = null);
         },
       ),
       transformTargetId: target,
@@ -90,14 +126,23 @@ class _VocabChips extends StatelessWidget {
 
   @override
   Widget build(BuildContext context) {
+    final tokens = widget.vocab.map((v) => v.asToken()).toList();
+    final newTokens = TokensUtil.getNewTokens("activity_tokens", tokens);
+    final renderer = TokenRenderingUtil(
+      existingStyle: TextStyle(
+        color: Theme.of(context).colorScheme.onSurface,
+        fontSize: 14.0,
+      ),
+    );
+
     return Wrap(
       spacing: 4.0,
       runSpacing: 4.0,
       children: [
-        ...vocab.map(
+        ...widget.vocab.map(
           (v) {
-            final target = "$targetId-${v.lemma}";
-            final color = usedVocab.contains(v.lemma.toLowerCase())
+            final target = "${widget.targetId}-${v.lemma}";
+            final color = widget.usedVocab.contains(v.lemma.toLowerCase())
                 ? Color.alphaBlend(
                     Theme.of(context).colorScheme.surface.withAlpha(150),
                     AppConfig.gold,
@@ -105,6 +150,8 @@ class _VocabChips extends StatelessWidget {
                 : Theme.of(context).colorScheme.primary.withAlpha(20);
 
             final linkAndKey = MatrixState.pAnyState.layerLinkAndKey(target);
+            final isNew = newTokens
+                .any((t) => t.content.toLowerCase() == v.lemma.toLowerCase());
 
             return CompositedTransformTarget(
               link: linkAndKey.link,
@@ -113,21 +160,28 @@ class _VocabChips extends StatelessWidget {
                 borderRadius: BorderRadius.circular(
                   24.0,
                 ),
-                onTap: () => _onTap(v, context),
-                child: Container(
-                  padding: const EdgeInsets.symmetric(
-                    horizontal: 8.0,
-                    vertical: 4.0,
-                  ),
-                  decoration: BoxDecoration(
-                    color: color,
-                    borderRadius: BorderRadius.circular(20),
-                  ),
-                  child: Text(
-                    v.lemma,
-                    style: TextStyle(
-                      color: Theme.of(context).colorScheme.onSurface,
-                      fontSize: 14.0,
+                onTap: () => _onTap(v, isNew),
+                child: HoverBuilder(
+                  builder: (context, hovered) => Container(
+                    padding: const EdgeInsets.symmetric(
+                      horizontal: 8.0,
+                      vertical: 4.0,
+                    ),
+                    decoration: BoxDecoration(
+                      color: color,
+                      borderRadius: BorderRadius.circular(20),
+                    ),
+                    child: Text(
+                      v.lemma,
+                      style: renderer.style(
+                        underlineColor: Theme.of(context)
+                            .colorScheme
+                            .primary
+                            .withAlpha(200),
+                        isNew: isNew,
+                        selected: _selectedVocab == v,
+                        hovered: hovered,
+                      ),
                     ),
                   ),
                 ),

lib/pangea/toolbar/message_selection_overlay.dart

@@ -11,9 +11,6 @@ import 'package:matrix/matrix.dart' hide Result;
 import 'package:fluffychat/config/app_config.dart';
 import 'package:fluffychat/pages/chat/chat.dart';
 import 'package:fluffychat/pangea/analytics_data/analytics_updater_mixin.dart';
-import 'package:fluffychat/pangea/analytics_misc/construct_type_enum.dart';
-import 'package:fluffychat/pangea/analytics_misc/construct_use_type_enum.dart';
-import 'package:fluffychat/pangea/analytics_misc/constructs_model.dart';
 import 'package:fluffychat/pangea/common/utils/error_handler.dart';
 import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart';
 import 'package:fluffychat/pangea/events/event_wrappers/pangea_representation_event.dart';
@@ -26,6 +23,7 @@ import 'package:fluffychat/pangea/toolbar/message_practice/practice_controller.d
 import 'package:fluffychat/pangea/toolbar/reading_assistance/select_mode_buttons.dart';
 import 'package:fluffychat/pangea/toolbar/reading_assistance/select_mode_controller.dart';
 import 'package:fluffychat/pangea/toolbar/reading_assistance/tokens_util.dart';
+import 'package:fluffychat/pangea/toolbar/token_rendering_mixin.dart';
 import 'package:fluffychat/widgets/matrix.dart';
 
 /// Controls data at the top level of the toolbar (mainly token / toolbar mode selection)
@@ -56,7 +54,7 @@ class MessageSelectionOverlay extends StatefulWidget {
 }
 
 class MessageOverlayController extends State<MessageSelectionOverlay>
-    with SingleTickerProviderStateMixin, AnalyticsUpdater {
+    with SingleTickerProviderStateMixin, AnalyticsUpdater, TokenRenderingMixin {
   Event get event => widget._event;
 
   PangeaTokenText? _selectedSpan;
@@ -218,27 +216,13 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
 
     if (!mounted) return;
     if (selectedToken != null && isNewToken(selectedToken!)) {
-      TokensUtil.collectToken(event.eventId, selectedToken!.text);
       final token = selectedToken!;
-      final constructs = [
-        OneConstructUse(
-          useType: ConstructUseTypeEnum.click,
-          lemma: token.lemma.text,
-          constructType: ConstructTypeEnum.vocab,
-          metadata: ConstructUseMetaData(
-            roomId: event.room.id,
-            timeStamp: DateTime.now(),
-            eventId: event.eventId,
-          ),
-          category: token.pos,
-          form: token.text.content,
-          xp: ConstructUseTypeEnum.click.pointValue,
-        ),
-      ];
-
-      addAnalytics(constructs, "word-zoom-card-${token.text.uniqueKey}")
-          .then((_) {
-        TokensUtil.clearNewTokenCache();
+      collectNewToken(
+        event.eventId,
+        "word-zoom-card-${token.text.uniqueKey}",
+        token,
+        Matrix.of(context).analyticsDataService,
+      ).then((_) {
         if (mounted) setState(() {});
       });
       return;
@@ -317,7 +301,7 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
   }
 
   bool isNewToken(PangeaToken token) =>
-      TokensUtil.isNewToken(token, pangeaMessageEvent);
+      TokensUtil.isNewTokenByEvent(token, pangeaMessageEvent);
 
   bool isTokenHighlighted(PangeaToken token) {
     if (_highlightedTokens == null) return false;

lib/pangea/toolbar/reading_assistance/token_rendering_util.dart

@@ -1,52 +1,45 @@
 import 'package:flutter/material.dart';
 
 import 'package:fluffychat/config/app_config.dart';
-import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart';
-import 'package:fluffychat/pangea/toolbar/layout/reading_assistance_mode_enum.dart';
-import 'package:fluffychat/pangea/toolbar/message_selection_overlay.dart';
 
 class TokenRenderingUtil {
-  final PangeaMessageEvent? pangeaMessageEvent;
-  final ReadingAssistanceMode? readingAssistanceMode;
-  final MessageOverlayController? overlayController;
-  final bool isTransitionAnimation;
   final TextStyle existingStyle;
 
+  TokenRenderingUtil({
+    required this.existingStyle,
+  });
+
   static final Map<String, double> _tokensWidthCache = {};
 
-  TokenRenderingUtil({
-    required this.pangeaMessageEvent,
-    required this.readingAssistanceMode,
-    required this.existingStyle,
-    this.overlayController,
-    this.isTransitionAnimation = false,
-  });
-
-  bool get showCenterStyling {
-    if (overlayController == null) return false;
-    if (!isTransitionAnimation) return true;
-    return readingAssistanceMode == ReadingAssistanceMode.practiceMode;
-  }
-
-  double? fontSize(BuildContext context) => showCenterStyling
-      ? overlayController != null && overlayController!.maxWidth > 600
-          ? Theme.of(context).textTheme.titleLarge?.fontSize
-          : Theme.of(context).textTheme.bodyLarge?.fontSize
-      : null;
-
-  TextStyle style(
-    BuildContext context, {
-    Color? color,
+  TextStyle style({
+    required Color underlineColor,
+    double? fontSize,
+    bool selected = false,
+    bool highlighted = false,
+    bool isNew = false,
+    bool practiceMode = false,
+    bool hovered = false,
   }) =>
       existingStyle.copyWith(
-        fontSize: fontSize(context),
+        fontSize: fontSize,
         decoration: TextDecoration.underline,
         decorationThickness: 4,
-        decorationColor: color ?? Colors.white.withAlpha(0),
+        decorationColor: _underlineColor(
+          underlineColor,
+          selected: selected,
+          highlighted: highlighted,
+          isNew: isNew,
+          practiceMode: practiceMode,
+          hovered: hovered,
+        ),
       );
 
-  double tokenTextWidthForContainer(BuildContext context, String text) {
-    final tokenSizeKey = "$text-${fontSize(context)}";
+  double tokenTextWidthForContainer(
+    String text,
+    Color underlineColor, {
+    double? fontSize,
+  }) {
+    final tokenSizeKey = "$text-$fontSize";
     if (_tokensWidthCache.containsKey(tokenSizeKey)) {
       return _tokensWidthCache[tokenSizeKey]!;
     }
@@ -54,7 +47,10 @@ class TokenRenderingUtil {
     final textPainter = TextPainter(
       text: TextSpan(
         text: text,
-        style: style(context),
+        style: style(
+          underlineColor: underlineColor,
+          fontSize: fontSize,
+        ),
       ),
       maxLines: 1,
       textDirection: TextDirection.ltr,
@@ -66,40 +62,19 @@ class TokenRenderingUtil {
     return width;
   }
 
-  // Only one token on the screen can have the token's unique key at a time.
-  // When readingAssistanceMode is not null, there are two messages - the centered message and the transition message.
-  // When in word mode, the key goes to the transition message.
-  // If actively transitioning, neither gets the keys.
-  // If in message mode, the key goes to the centered message (isTransitionAnimation == false).
-  bool get assignTokenKey {
-    if (readingAssistanceMode == null) {
-      return false;
-    }
-
-    switch (readingAssistanceMode!) {
-      case ReadingAssistanceMode.selectMode:
-        return isTransitionAnimation;
-      case ReadingAssistanceMode.transitionMode:
-        return false;
-      case ReadingAssistanceMode.practiceMode:
-        return !isTransitionAnimation;
-    }
-  }
-
-  Color backgroundColor(
-    BuildContext context,
-    bool selected,
-    bool highlighted,
-    bool isNew,
-    bool practiceMode,
-  ) {
+  Color _underlineColor(
+    Color underlineColor, {
+    bool selected = false,
+    bool highlighted = false,
+    bool isNew = false,
+    bool practiceMode = false,
+    bool hovered = false,
+  }) {
     if (practiceMode) return Colors.white.withAlpha(0);
-    if (highlighted) {
-      return Theme.of(context).colorScheme.primary.withAlpha(200);
-    }
+    if (highlighted) return underlineColor;
     if (isNew) return AppConfig.success.withAlpha(200);
-    return selected
-        ? Theme.of(context).colorScheme.primary.withAlpha(200)
-        : Colors.white.withAlpha(0);
+    if (selected) return underlineColor;
+    if (hovered) return underlineColor.withAlpha(100);
+    return Colors.white.withAlpha(0);
   }
 }

lib/pangea/toolbar/reading_assistance/tokens_util.dart

@@ -51,11 +51,11 @@ class TokensUtil {
 
   static const Duration _cacheDuration = Duration(minutes: 1);
 
-  static List<PangeaTokenText>? _getCachedNewTokens(String eventID) {
-    final cacheItem = _newTokenCache[eventID];
+  static List<PangeaTokenText>? _getCachedNewTokens(String cacheKey) {
+    final cacheItem = _newTokenCache[cacheKey];
     if (cacheItem == null) return null;
     if (cacheItem.timestamp.isBefore(DateTime.now().subtract(_cacheDuration))) {
-      _newTokenCache.remove(eventID);
+      _newTokenCache.remove(cacheKey);
       return null;
     }
 
@@ -63,16 +63,52 @@ class TokensUtil {
   }
 
   static void _setCachedNewTokens(
-    String eventID,
+    String cacheKey,
     List<PangeaTokenText> tokens,
   ) {
-    _newTokenCache[eventID] = _NewTokenCacheItem(
+    _newTokenCache[cacheKey] = _NewTokenCacheItem(
       tokens,
       DateTime.now(),
     );
   }
 
   static List<PangeaTokenText> getNewTokens(
+    String cacheKey,
+    List<PangeaToken> tokens, {
+    int? maxTokens,
+  }) {
+    if (MatrixState
+        .pangeaController.matrixState.analyticsDataService.isInitializing) {
+      return [];
+    }
+
+    final cached = _getCachedNewTokens(cacheKey);
+    if (cached != null) return cached;
+
+    final List<PangeaTokenText> newTokens = [];
+    final analyticsService =
+        MatrixState.pangeaController.matrixState.analyticsDataService;
+
+    for (final token in tokens) {
+      if (!token.lemma.saveVocab || !token.vocabConstructID.isContentWord) {
+        continue;
+      }
+
+      if (analyticsService.hasUsedConstruct(token.vocabConstructID)) {
+        continue;
+      }
+
+      if (newTokens.any((t) => t == token.text)) continue;
+
+      newTokens.add(token.text);
+      if (maxTokens != null && newTokens.length >= maxTokens) break;
+    }
+
+    _setCachedNewTokens(cacheKey, newTokens);
+    return newTokens;
+  }
+
+  static List<PangeaTokenText> getNewTokensByEvent(
     PangeaMessageEvent event,
   ) {
     if (!event.eventId.isValidMatrixId ||
@@ -100,30 +136,19 @@ class TokensUtil {
       return [];
     }
 
-    final List<PangeaTokenText> newTokens = [];
-    final analyticsService =
-        MatrixState.pangeaController.matrixState.analyticsDataService;
-    for (final token in tokens) {
-      if (!token.lemma.saveVocab || !token.vocabConstructID.isContentWord) {
-        continue;
-      }
-
-      if (analyticsService.hasUsedConstruct(token.vocabConstructID)) {
-        continue;
-      }
-
-      if (newTokens.any((t) => t == token.text)) continue;
-
-      newTokens.add(token.text);
-      if (newTokens.length >= 3) break;
-    }
-
-    _setCachedNewTokens(event.eventId, newTokens);
-    return newTokens;
+    return getNewTokens(event.eventId, tokens, maxTokens: 3);
   }
 
-  static bool isNewToken(PangeaToken token, PangeaMessageEvent event) {
-    final newTokens = getNewTokens(event);
+  static bool isNewToken(
+    String cacheKey,
+    PangeaToken token,
+  ) {
+    final newTokens = getNewTokens(cacheKey, [token]);
+    return newTokens.any((t) => t == token.text);
+  }
+
+  static bool isNewTokenByEvent(PangeaToken token, PangeaMessageEvent event) {
+    final newTokens = getNewTokensByEvent(event);
     return newTokens.any((t) => t == token.text);
   }
 
@@ -131,8 +156,8 @@ class TokensUtil {
     _newTokenCache.clear();
   }
 
-  static void collectToken(String eventId, PangeaTokenText token) {
-    _newTokenCache[eventId]?.tokens.remove(token);
+  static void collectToken(String cachedKey, PangeaTokenText token) {
+    _newTokenCache[cachedKey]?.tokens.remove(token);
     _lastCollected = token;
   }
 
@@ -141,11 +166,11 @@ class TokensUtil {
 
   static void clearRecentlyCollected() => _lastCollected = null;
 
-  static List<TokenPosition>? _getCachedTokenPositions(String eventID) {
-    final cacheItem = _tokenPositionCache[eventID];
+  static List<TokenPosition>? _getCachedTokenPositions(String cacheKey) {
+    final cacheItem = _tokenPositionCache[cacheKey];
     if (cacheItem == null) return null;
     if (cacheItem.timestamp.isBefore(DateTime.now().subtract(_cacheDuration))) {
-      _tokenPositionCache.remove(eventID);
+      _tokenPositionCache.remove(cacheKey);
       return null;
     }
 
@@ -153,10 +178,10 @@ class TokensUtil {
   }
 
   static void _setCachedTokenPositions(
-    String eventID,
+    String cacheKey,
     List<TokenPosition> positions,
   ) {
-    _tokenPositionCache[eventID] = _TokenPositionCacheItem(
+    _tokenPositionCache[cacheKey] = _TokenPositionCacheItem(
       positions,
       DateTime.now(),
     );

lib/pangea/toolbar/token_rendering_mixin.dart

@@ -0,0 +1,40 @@
+import 'package:fluffychat/pangea/analytics_data/analytics_data_service.dart';
+import 'package:fluffychat/pangea/analytics_misc/construct_type_enum.dart';
+import 'package:fluffychat/pangea/analytics_misc/construct_use_type_enum.dart';
+import 'package:fluffychat/pangea/analytics_misc/constructs_model.dart';
+import 'package:fluffychat/pangea/events/models/pangea_token_model.dart';
+import 'package:fluffychat/pangea/toolbar/reading_assistance/tokens_util.dart';
+
+mixin TokenRenderingMixin {
+  Future<void> collectNewToken(
+    String cacheKey,
+    String targetId,
+    PangeaToken token,
+    AnalyticsDataService analyticsService, {
+    String? roomId,
+    String? eventId,
+  }) async {
+    TokensUtil.collectToken(cacheKey, token.text);
+    final constructs = [
+      OneConstructUse(
+        useType: ConstructUseTypeEnum.click,
+        lemma: token.lemma.text,
+        constructType: ConstructTypeEnum.vocab,
+        metadata: ConstructUseMetaData(
+          roomId: roomId,
+          timeStamp: DateTime.now(),
+          eventId: eventId,
+        ),
+        category: token.pos,
+        form: token.text.content,
+        xp: ConstructUseTypeEnum.click.pointValue,
+      ),
+    ];
+
+    await analyticsService.updateService.addAnalytics(
+      targetId,
+      constructs,
+    );
+    TokensUtil.clearNewTokenCache();
+  }
+}