Highlight audio text (#1333)

* Highlight text as TTS plays - attempt 1

* Make highlighting actually work

* Fix to minor version of punctuation issue

* Highlights all applicable text

* fix: filter out punctuation tokens in the client side when highlighing audio tokens

* Highlight selection separate from normal selection

* cleanup: further decouple tts highlighting and token selection, renamed temporarySelection => _highlightedTokens

* fix: don't show token highlights for non-overlay messages

---------

Co-authored-by: ggurdin <46800240+ggurdin@users.noreply.github.com>
Co-authored-by: ggurdin <ggurdin@gmail.com>

lib/pages/chat/events/audio_player.dart

@@ -13,6 +13,7 @@ import 'package:path_provider/path_provider.dart';
 
 import 'package:fluffychat/config/app_config.dart';
 import 'package:fluffychat/pangea/toolbar/widgets/message_audio_card.dart';
+import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart';
 import 'package:fluffychat/utils/error_reporter.dart';
 import 'package:fluffychat/utils/localized_exception_extension.dart';
 import 'package:fluffychat/utils/url_launcher.dart';
@@ -28,6 +29,7 @@ class AudioPlayerWidget extends StatefulWidget {
   final bool autoplay;
   final Function(bool)? setIsPlayingAudio;
   final double padding;
+  final MessageOverlayController? overlayController;
   // Pangea#
 
   static String? currentId;
@@ -50,6 +52,7 @@ class AudioPlayerWidget extends StatefulWidget {
     this.sectionEndMS,
     this.setIsPlayingAudio,
     this.padding = 12.0,
+    this.overlayController,
     // Pangea#
     super.key,
   });
@@ -187,6 +190,15 @@ class AudioPlayerState extends State<AudioPlayerWidget> {
         audioPlayer.stop();
         audioPlayer.seek(null);
       }
+      // #Pangea
+      // Pass current timestamp to overlay, so it can highlight as necessary
+      if (widget.matrixFile != null) {
+        widget.overlayController?.highlightCurrentText(
+          state.inMilliseconds,
+          widget.matrixFile!.tokens,
+        );
+      }
+      // Pangea#
     });
     onDurationChanged ??= audioPlayer.durationStream.listen((max) {
       if (max == null || max == Duration.zero) return;

lib/pages/chat/events/message_content.dart

@@ -337,7 +337,12 @@ class MessageContent extends StatelessWidget {
                     selectedToken: token,
                   );
                 },
-                isSelected: overlayController?.isTokenSelected,
+                isSelected: overlayController != null
+                    ? (token) {
+                        return overlayController!.isTokenSelected(token) ||
+                            overlayController!.isTokenHighlighted(token);
+                      }
+                    : null,
               );
             }
 

lib/pangea/toolbar/repo/lemma_activity_generator.dart

@@ -64,7 +64,7 @@ class LemmaActivityGenerator {
     final choices = sortedLemmas.take(4).toList();
     if (!choices.contains(token.lemma.text)) {
       final random = Random();
-      choices[random.nextInt(4)] = token.lemma.text;
+      choices[random.nextInt(choices.length - 1)] = token.lemma.text;
     }
     return choices;
   }

lib/pangea/toolbar/widgets/message_audio_card.dart

@@ -201,6 +201,7 @@ class MessageAudioCardState extends State<MessageAudioCard> {
                       fontSize:
                           AppConfig.messageFontSize * AppConfig.fontSizeFactor,
                       padding: 0,
+                      overlayController: widget.overlayController,
                     )
                   : const CardErrorWidget(
                       error: "Null audio file in message_audio_card",

lib/pangea/toolbar/widgets/message_selection_overlay.dart

@@ -17,6 +17,7 @@ import 'package:fluffychat/pangea/common/utils/error_handler.dart';
 import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart';
 import 'package:fluffychat/pangea/events/models/pangea_token_model.dart';
 import 'package:fluffychat/pangea/events/models/pangea_token_text_model.dart';
+import 'package:fluffychat/pangea/toolbar/controllers/text_to_speech_controller.dart';
 import 'package:fluffychat/pangea/toolbar/enums/activity_type_enum.dart';
 import 'package:fluffychat/pangea/toolbar/enums/message_mode_enum.dart';
 import 'package:fluffychat/pangea/toolbar/widgets/message_toolbar.dart';
@@ -61,6 +62,7 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
 
   MessageMode toolbarMode = MessageMode.noneSelected;
   PangeaTokenText? _selectedSpan;
+  List<PangeaTokenText>? _highlightedTokens;
 
   List<PangeaToken>? tokens;
   bool initialized = false;
@@ -129,6 +131,26 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
     setState(() {});
   }
 
+  /// If sentence TTS is playing a word, highlight that word in message overlay
+  void highlightCurrentText(int currentPosition, List<TTSToken> ttsTokens) {
+    final List<TTSToken> textToSelect = [];
+    // Check if current time is between start and end times of tokens
+    for (final TTSToken token in ttsTokens) {
+      if (token.endMS > currentPosition) {
+        if (token.startMS < currentPosition) {
+          textToSelect.add(token);
+        } else {
+          break;
+        }
+      }
+    }
+
+    if (const ListEquality().equals(textToSelect, _highlightedTokens)) return;
+    _highlightedTokens =
+        textToSelect.isEmpty ? null : textToSelect.map((t) => t.text).toList();
+    setState(() {});
+  }
+
   void _setupSubscriptions() {
     _animationController = AnimationController(
       vsync: this,
@@ -278,6 +300,9 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
         debugPrint("updateToolbarMode: $mode - clearing selectedSpan");
         _selectedSpan = null;
       }
+      if (mode != MessageMode.textToSpeech) {
+        _highlightedTokens = null;
+      }
       toolbarMode = mode;
     });
   }
@@ -327,17 +352,24 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
     setState(() {});
   }
 
-  /// Whether the given token is currently selected
+  /// Whether the given token is currently selected or highlighted
   bool isTokenSelected(PangeaToken token) {
     final isSelected = _selectedSpan?.offset == token.text.offset &&
         _selectedSpan?.length == token.text.length;
     return isSelected;
   }
 
+  bool isTokenHighlighted(PangeaToken token) {
+    if (_highlightedTokens == null) return false;
+    return _highlightedTokens!.any(
+      (t) => t.offset == token.text.offset && t.length == token.text.length,
+    );
+  }
+
   PangeaToken? get selectedToken => tokens?.firstWhereOrNull(isTokenSelected);
 
   /// Whether the overlay is currently displaying a selection
-  bool get isSelection => _selectedSpan != null;
+  bool get isSelection => _selectedSpan != null || _highlightedTokens != null;
 
   PangeaTokenText? get selectedSpan => _selectedSpan;