diff --git a/lib/pangea/analytics_details_popup/vocab_analytics_details_view.dart b/lib/pangea/analytics_details_popup/vocab_analytics_details_view.dart index 9446dfb1b..de1c909b3 100644 --- a/lib/pangea/analytics_details_popup/vocab_analytics_details_view.dart +++ b/lib/pangea/analytics_details_popup/vocab_analytics_details_view.dart @@ -28,9 +28,6 @@ class VocabDetailsView extends StatelessWidget { ConstructUses get _construct => constructId.constructUses; - String? get _userL1 => - MatrixState.pangeaController.languageController.userL1?.langCode; - /// Get the language code for the current lemma String? get _userL2 => MatrixState.pangeaController.languageController.userL2?.langCode; diff --git a/lib/pangea/choreographer/controllers/igc_controller.dart b/lib/pangea/choreographer/controllers/igc_controller.dart index 8af6dc802..3e7cd3ac3 100644 --- a/lib/pangea/choreographer/controllers/igc_controller.dart +++ b/lib/pangea/choreographer/controllers/igc_controller.dart @@ -256,14 +256,7 @@ class IgcController { timeline: choreographer.chatController.timeline!, ownMessage: event.senderId == choreographer.pangeaController.matrixState.client.userID, - ) - .getSpeechToTextLocal( - choreographer.l1LangCode, - choreographer.l2LangCode, - ) - ?.transcript - .text - .trim(); // trim whitespace + ).getSpeechToTextLocal()?.transcript.text.trim(); // trim whitespace if (content == null) continue; messages.add( PreviousMessage( diff --git a/lib/pangea/events/event_wrappers/pangea_message_event.dart b/lib/pangea/events/event_wrappers/pangea_message_event.dart index d76519f38..0f01f2655 100644 --- a/lib/pangea/events/event_wrappers/pangea_message_event.dart +++ b/lib/pangea/events/event_wrappers/pangea_message_event.dart @@ -232,13 +232,7 @@ class PangeaMessageEvent { null; }).toSet(); - SpeechToTextModel? getSpeechToTextLocal( - String? l1Code, - String? l2Code, - ) { - if (l1Code == null || l2Code == null) { - return null; - } + SpeechToTextModel? getSpeechToTextLocal() { return representations .firstWhereOrNull( (element) => element.content.speechToText != null, diff --git a/lib/pangea/lemmas/lemma_reaction_picker.dart b/lib/pangea/lemmas/lemma_reaction_picker.dart index 92a3a70ef..3b157c2b7 100644 --- a/lib/pangea/lemmas/lemma_reaction_picker.dart +++ b/lib/pangea/lemmas/lemma_reaction_picker.dart @@ -53,7 +53,7 @@ class LemmaReactionPickerState extends State { } catch (e, s) { ErrorHandler.logError(data: widget.cId.toJson(), e: e, s: s); } finally { - setState(() => loading = false); + if (mounted) setState(() => loading = false); } } diff --git a/lib/pangea/message_token_text/token_position_model.dart b/lib/pangea/message_token_text/token_position_model.dart index a01434078..5e8c9b57b 100644 --- a/lib/pangea/message_token_text/token_position_model.dart +++ b/lib/pangea/message_token_text/token_position_model.dart @@ -1,29 +1,85 @@ import 'package:fluffychat/pangea/events/models/pangea_token_model.dart'; -class TokenPositionModel { - /// Start index of the full substring in the message - final int start; - - /// End index of the full substring in the message - final int end; - - /// Start index of the token in the message - final int tokenStart; - - /// End index of the token in the message - final int tokenEnd; - - final bool selected; - final bool hideContent; +class TokenPosition { final PangeaToken? token; + final int startIndex; + final int endIndex; - const TokenPositionModel({ - required this.start, - required this.end, - required this.tokenStart, - required this.tokenEnd, - required this.hideContent, - required this.selected, + const TokenPosition({ this.token, + required this.startIndex, + required this.endIndex, }); } + +class TokensUtil { + static List getTokenPositions( + List tokens, + ) { + final List tokenPositions = []; + int tokenPointer = 0; + int globalPointer = 0; + + while (tokenPointer < tokens.length) { + int endIndex = tokenPointer; + PangeaToken token = tokens[tokenPointer]; + + if (token.text.offset > globalPointer) { + // If the token starts after the current global pointer, we need to + // create a new token position for the gap + tokenPositions.add( + TokenPosition( + startIndex: globalPointer, + endIndex: token.text.offset, + ), + ); + + globalPointer = token.text.offset; + } + + // move the end index if the next token is right next to the current token + // and either the current token is punctuation or the next token is punctuation + while (endIndex < tokens.length - 1) { + final PangeaToken currentToken = tokens[endIndex]; + final PangeaToken nextToken = tokens[endIndex + 1]; + + final currentIsPunct = currentToken.pos == 'PUNCT' && + currentToken.text.content.trim().isNotEmpty; + final nextIsPunct = nextToken.pos == 'PUNCT' && + nextToken.text.content.trim().isNotEmpty; + + if (currentToken.text.offset + currentToken.text.length != + nextToken.text.offset) { + break; + } + + if ((currentIsPunct && nextIsPunct) || + (currentIsPunct && nextToken.text.content.trim().isNotEmpty) || + (nextIsPunct && currentToken.text.content.trim().isNotEmpty)) { + if (token.pos == 'PUNCT' && !nextIsPunct) { + token = nextToken; + } + + endIndex++; + } else { + break; + } + } + + tokenPositions.add( + TokenPosition( + token: token, + startIndex: tokens[tokenPointer].text.offset, + endIndex: tokens[endIndex].text.offset + tokens[endIndex].text.length, + ), + ); + + // Move to the next token + tokenPointer = tokenPointer + (endIndex - tokenPointer) + 1; + globalPointer = + tokens[endIndex].text.offset + tokens[endIndex].text.length; + } + + return tokenPositions; + } +} diff --git a/lib/pangea/toolbar/widgets/message_selection_overlay.dart b/lib/pangea/toolbar/widgets/message_selection_overlay.dart index cdacaa4bb..daaecd2ae 100644 --- a/lib/pangea/toolbar/widgets/message_selection_overlay.dart +++ b/lib/pangea/toolbar/widgets/message_selection_overlay.dart @@ -461,9 +461,18 @@ class MessageOverlayController extends State pangeaMessageEvent?.messageDisplayLangCode.split("-")[0] == MatrixState.pangeaController.languageController.userL2?.langCodeShort; - PangeaToken? get selectedToken => - pangeaMessageEvent?.messageDisplayRepresentation?.tokens - ?.firstWhereOrNull(isTokenSelected); + PangeaToken? get selectedToken { + if (pangeaMessageEvent?.isAudioMessage == true) { + final stt = pangeaMessageEvent!.getSpeechToTextLocal(); + if (stt == null || stt.transcript.sttTokens.isEmpty) return null; + return stt.transcript.sttTokens + .firstWhereOrNull((t) => isTokenSelected(t.token)) + ?.token; + } + + return pangeaMessageEvent?.messageDisplayRepresentation?.tokens + ?.firstWhereOrNull(isTokenSelected); + } /// Whether the overlay is currently displaying a selection bool get isSelection => _selectedSpan != null || _highlightedTokens != null; diff --git a/lib/pangea/toolbar/widgets/overlay_message.dart b/lib/pangea/toolbar/widgets/overlay_message.dart index 14dea0f1c..fe783c503 100644 --- a/lib/pangea/toolbar/widgets/overlay_message.dart +++ b/lib/pangea/toolbar/widgets/overlay_message.dart @@ -15,6 +15,7 @@ import 'package:fluffychat/pangea/learning_settings/utils/p_language_store.dart' import 'package:fluffychat/pangea/phonetic_transcription/phonetic_transcription_widget.dart'; import 'package:fluffychat/pangea/toolbar/enums/reading_assistance_mode_enum.dart'; import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart'; +import 'package:fluffychat/pangea/toolbar/widgets/stt_transcript_tokens.dart'; import 'package:fluffychat/utils/date_time_extension.dart'; import 'package:fluffychat/utils/file_description.dart'; import 'package:fluffychat/widgets/matrix.dart'; @@ -177,15 +178,17 @@ class OverlayMessage extends StatelessWidget { spacing: 8.0, crossAxisAlignment: CrossAxisAlignment.start, children: [ - Text( - overlayController - .transcription!.transcript.text, + SttTranscriptTokens( + model: overlayController.transcription!, style: AppConfig.messageTextStyle( event, textColor, ).copyWith( fontStyle: FontStyle.italic, ), + onClick: overlayController + .onClickOverlayMessageToken, + isSelected: overlayController.isTokenSelected, ), if (MatrixState.pangeaController .languageController.showTrancription) diff --git a/lib/pangea/toolbar/widgets/stt_transcript_tokens.dart b/lib/pangea/toolbar/widgets/stt_transcript_tokens.dart new file mode 100644 index 000000000..98777076b --- /dev/null +++ b/lib/pangea/toolbar/widgets/stt_transcript_tokens.dart @@ -0,0 +1,91 @@ +import 'package:flutter/material.dart'; + +import 'package:fluffychat/pangea/events/models/pangea_token_model.dart'; +import 'package:fluffychat/pangea/message_token_text/token_position_model.dart'; +import 'package:fluffychat/pangea/toolbar/models/speech_to_text_models.dart'; +import 'package:fluffychat/widgets/matrix.dart'; + +class SttTranscriptTokens extends StatelessWidget { + final SpeechToTextModel model; + final TextStyle? style; + + final void Function(PangeaToken)? onClick; + final bool Function(PangeaToken)? isSelected; + + const SttTranscriptTokens({ + super.key, + required this.model, + this.onClick, + this.isSelected, + this.style, + }); + + List get tokens => + model.transcript.sttTokens.map((t) => t.token).toList(); + + @override + Widget build(BuildContext context) { + if (model.transcript.sttTokens.isEmpty) { + return Text( + model.transcript.text, + style: style ?? DefaultTextStyle.of(context).style, + textScaler: TextScaler.noScaling, + ); + } + + final messageCharacters = model.transcript.text.characters; + return RichText( + textScaler: TextScaler.noScaling, + text: TextSpan( + style: style ?? DefaultTextStyle.of(context).style, + children: TokensUtil.getTokenPositions(tokens).map((tokenPosition) { + final text = messageCharacters + .skip(tokenPosition.startIndex) + .take(tokenPosition.endIndex - tokenPosition.startIndex) + .toString(); + + if (tokenPosition.token == null) { + return TextSpan( + text: text, + style: style ?? DefaultTextStyle.of(context).style, + ); + } + + final token = tokenPosition.token!; + final selected = isSelected?.call(token) ?? false; + + return WidgetSpan( + child: CompositedTransformTarget( + link: MatrixState.pAnyState + .layerLinkAndKey(token.text.uniqueKey) + .link, + child: MouseRegion( + key: MatrixState.pAnyState + .layerLinkAndKey(token.text.uniqueKey) + .key, + cursor: SystemMouseCursors.click, + child: GestureDetector( + behavior: HitTestBehavior.translucent, + onTap: onClick != null ? () => onClick?.call(token) : null, + child: RichText( + text: TextSpan( + text: text, + style: (style ?? DefaultTextStyle.of(context).style) + .copyWith( + decoration: TextDecoration.underline, + decorationThickness: 4, + decorationColor: selected + ? Theme.of(context).colorScheme.primary + : Colors.white.withAlpha(0), + ), + ), + ), + ), + ), + ), + ); + }).toList(), + ), + ); + } +}