From 51b5ec1163d2d3bd7834d7e3f51bd6b13bc04d23 Mon Sep 17 00:00:00 2001 From: ggurdin <46800240+ggurdin@users.noreply.github.com> Date: Mon, 28 Apr 2025 10:22:32 -0400 Subject: [PATCH] chore: move STT into message bubble (#2564) --- .../reading_assistance_input_bar.dart | 10 +- .../widgets/message_selection_overlay.dart | 5 - .../widgets/message_selection_positioner.dart | 14 +- .../widgets/message_speech_to_text_card.dart | 204 +++++------------- .../toolbar/widgets/overlay_message.dart | 26 ++- 5 files changed, 79 insertions(+), 180 deletions(-) diff --git a/lib/pangea/toolbar/reading_assistance_input_row/reading_assistance_input_bar.dart b/lib/pangea/toolbar/reading_assistance_input_row/reading_assistance_input_bar.dart index 618a9e919..8b1d64e6e 100644 --- a/lib/pangea/toolbar/reading_assistance_input_row/reading_assistance_input_bar.dart +++ b/lib/pangea/toolbar/reading_assistance_input_row/reading_assistance_input_bar.dart @@ -9,7 +9,6 @@ import 'package:fluffychat/pangea/toolbar/enums/message_mode_enum.dart'; import 'package:fluffychat/pangea/toolbar/enums/reading_assistance_mode_enum.dart'; import 'package:fluffychat/pangea/toolbar/widgets/message_mode_locked_card.dart'; import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart'; -import 'package:fluffychat/pangea/toolbar/widgets/message_speech_to_text_card.dart'; import 'package:fluffychat/pangea/toolbar/widgets/message_translation_card.dart'; import 'package:fluffychat/pangea/toolbar/widgets/practice_activity/practice_activity_card.dart'; @@ -43,14 +42,7 @@ class ReadingAssistanceInputBar extends StatelessWidget { : null; if (overlayController.pangeaMessageEvent?.isAudioMessage == true) { - if (!['audio/wav', 'audio/x-wav'] - .contains(overlayController.pangeaMessageEvent!.mimetype)) { - return ReactionsPicker(controller); - } - - content = MessageSpeechToTextCard( - messageEvent: overlayController.pangeaMessageEvent!, - ); + return ReactionsPicker(controller); } else { switch (overlayController.toolbarMode) { case MessageMode.messageSpeechToText: diff --git a/lib/pangea/toolbar/widgets/message_selection_overlay.dart b/lib/pangea/toolbar/widgets/message_selection_overlay.dart index 284914c5a..10e772962 100644 --- a/lib/pangea/toolbar/widgets/message_selection_overlay.dart +++ b/lib/pangea/toolbar/widgets/message_selection_overlay.dart @@ -183,11 +183,6 @@ class MessageOverlayController extends State } Future _setInitialToolbarMode() async { - if (pangeaMessageEvent?.isAudioMessage ?? false) { - updateToolbarMode(MessageMode.listening); - return; - } - // 1) if we have a hidden word activity, then we should start with that if (practiceSelection?.hasHiddenWordActivity ?? false) { updateToolbarMode(MessageMode.practiceActivity); diff --git a/lib/pangea/toolbar/widgets/message_selection_positioner.dart b/lib/pangea/toolbar/widgets/message_selection_positioner.dart index e612379eb..ded3cc757 100644 --- a/lib/pangea/toolbar/widgets/message_selection_positioner.dart +++ b/lib/pangea/toolbar/widgets/message_selection_positioner.dart @@ -484,19 +484,23 @@ class MessageSelectionPositionerState extends State // measurement for items in the toolbar - bool get showPracticeButtons => + bool get _showButtons => (widget.pangeaMessageEvent?.shouldShowToolbar ?? false) && widget.pangeaMessageEvent?.event.messageType == MessageTypes.Text && - (widget.pangeaMessageEvent?.messageDisplayLangIsL2 ?? false) && + (widget.pangeaMessageEvent?.messageDisplayLangIsL2 ?? false); + + bool get showPracticeButtons => + _showButtons && widget.overlayController.readingAssistanceMode == ReadingAssistanceMode.practiceMode; bool get showSelectionButtons => - widget.overlayController.readingAssistanceMode == - ReadingAssistanceMode.selectMode; + _showButtons && + [ReadingAssistanceMode.selectMode, null] + .contains(widget.overlayController.readingAssistanceMode); double get _selectionButtonsHeight { - return AppConfig.toolbarButtonsHeight; + return showSelectionButtons ? AppConfig.toolbarButtonsHeight : 0; } bool get _hasReactions { diff --git a/lib/pangea/toolbar/widgets/message_speech_to_text_card.dart b/lib/pangea/toolbar/widgets/message_speech_to_text_card.dart index 27a1d5e35..49cb775c4 100644 --- a/lib/pangea/toolbar/widgets/message_speech_to_text_card.dart +++ b/lib/pangea/toolbar/widgets/message_speech_to_text_card.dart @@ -6,23 +6,19 @@ import 'package:flutter/material.dart'; import 'package:flutter_gen/gen_l10n/l10n.dart'; import 'package:fluffychat/config/app_config.dart'; -import 'package:fluffychat/pangea/choreographer/widgets/igc/card_error_widget.dart'; import 'package:fluffychat/pangea/common/utils/error_handler.dart'; import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart'; -import 'package:fluffychat/pangea/instructions/instructions_enum.dart'; -import 'package:fluffychat/pangea/instructions/instructions_inline_tooltip.dart'; import 'package:fluffychat/pangea/toolbar/models/speech_to_text_models.dart'; -import 'package:fluffychat/pangea/toolbar/widgets/icon_number_widget.dart'; -import 'package:fluffychat/pangea/toolbar/widgets/toolbar_content_loading_indicator.dart'; import 'package:fluffychat/widgets/matrix.dart'; -import '../../bot/utils/bot_style.dart'; class MessageSpeechToTextCard extends StatefulWidget { final PangeaMessageEvent messageEvent; + final Color textColor; const MessageSpeechToTextCard({ super.key, required this.messageEvent, + required this.textColor, }); @override @@ -30,29 +26,33 @@ class MessageSpeechToTextCard extends StatefulWidget { } class MessageSpeechToTextCardState extends State { - SpeechToTextModel? speechToTextResponse; + SpeechToTextModel? _speechToTextResponse; + bool _fetchingTranscription = true; Object? error; - STTToken? selectedToken; - TextSpan? transcriptText; String? get l1Code => MatrixState.pangeaController.languageController.activeL1Code(); String? get l2Code => MatrixState.pangeaController.languageController.activeL2Code(); + @override + void initState() { + super.initState(); + _fetchTranscription(); + } + // look for transcription in message event // if not found, call API to transcribe audio - Future getSpeechToText() async { + Future _fetchTranscription() async { try { if (l1Code == null || l2Code == null) { throw Exception('Language selection not found'); } - speechToTextResponse ??= - await widget.messageEvent.getSpeechToText(l1Code!, l2Code!); - debugPrint( - 'Speech to text transcript: ${speechToTextResponse?.transcript.text}', + _speechToTextResponse ??= await widget.messageEvent.getSpeechToText( + l1Code!, + l2Code!, ); } catch (e, s) { debugger(when: kDebugMode); @@ -69,153 +69,51 @@ class MessageSpeechToTextCardState extends State { } } - TextSpan _buildTranscriptText(BuildContext context) { - try { - final Transcript transcript = speechToTextResponse!.transcript; - final List spans = []; - String remainingFullText = transcript.text; - - if (transcript.sttTokens.isEmpty) { - return TextSpan( - text: remainingFullText, - style: BotStyle.text(context), - ); - } - - for (final token in transcript.sttTokens) { - final offset = remainingFullText.indexOf(token.token.text.content); - if (offset == -1) continue; - final length = token.length; - - if (remainingFullText.substring(0, offset).trim().isNotEmpty) { - remainingFullText = remainingFullText.substring(offset); - continue; - } - - if (offset > 0) { - // Add any plain text before the token - spans.add( - TextSpan(text: remainingFullText.substring(0, offset)), - ); - } - - spans.add( - TextSpan( - text: remainingFullText.substring( - offset, - offset + token.length, - ), - style: BotStyle.text( - context, - existingStyle: TextStyle(color: token.color(context)), - setColor: false, - ), - // gesturRecognizer that sets selectedToken on click - // recognizer: TapGestureRecognizer() - // ..onTap = () { - // debugPrint('Token tapped'); - // debugPrint(token.toJson().toString()); - // if (mounted) { - // setState(() { - // if (selectedToken == token) { - // selectedToken = null; - // } else { - // selectedToken = token; - // } - // }); - // } - // }, - ), - ); - - remainingFullText = remainingFullText.substring(offset + length); - } - - if (remainingFullText.isNotEmpty) { - // Add any remaining text after the last token - spans.add(TextSpan(text: remainingFullText)); - } - - return TextSpan(children: spans); - } catch (err, s) { - ErrorHandler.logError( - e: err, - s: s, - data: {}, - ); - setState(() => error = err); - return const TextSpan(text: ''); - } - } - - @override - void initState() { - super.initState(); - getSpeechToText().then((_) { - if (mounted) { - setState(() => transcriptText = _buildTranscriptText(context)); - } - }); - } - - String? get wordsPerMinuteString => - speechToTextResponse?.transcript.wordsPerMinute?.toStringAsFixed(2); - @override Widget build(BuildContext context) { if (_fetchingTranscription) { - return const ToolbarContentLoadingIndicator(); + return const LinearProgressIndicator(); } - // done fetchig but not results means some kind of error - if (speechToTextResponse == null || error != null) { - return CardErrorWidget( - error: error ?? "Failed to fetch speech to text", - maxWidth: AppConfig.toolbarMinWidth, + // // done fetching but not results means some kind of error + if (_speechToTextResponse == null || error != null) { + return Row( + spacing: 8.0, + children: [ + Flexible( + child: RichText( + text: TextSpan( + style: AppConfig.messageTextStyle( + widget.messageEvent.event, + widget.textColor, + ), + children: [ + WidgetSpan( + alignment: PlaceholderAlignment.middle, + child: Icon( + Icons.error, + color: Theme.of(context).colorScheme.error, + ), + ), + const TextSpan(text: " "), + TextSpan( + text: L10n.of(context).oopsSomethingWentWrong, + ), + ], + ), + ), + ), + ], ); } - //TODO: find better icons - return ConstrainedBox( - constraints: const BoxConstraints( - maxWidth: AppConfig.toolbarMinWidth, - maxHeight: AppConfig.toolbarMaxHeight, - ), - child: SingleChildScrollView( - child: Padding( - padding: const EdgeInsets.all(16), - child: Column( - mainAxisSize: MainAxisSize.min, - children: [ - const SizedBox(height: 8), - RichText( - text: transcriptText!, - ), - if (widget.messageEvent.senderId == - Matrix.of(context).client.userID) - Column( - children: [ - const SizedBox(height: 16), - Row( - mainAxisSize: MainAxisSize.min, - children: [ - IconNumberWidget( - icon: Icons.speed, - number: wordsPerMinuteString != null - ? "$wordsPerMinuteString" - : "??", - toolTip: L10n.of(context).wordsPerMinute, - ), - ], - ), - const InstructionsInlineTooltip( - instructionsEnum: InstructionsEnum.speechToText, - ), - ], - ), - ], - ), - ), + return Text( + "${_speechToTextResponse?.transcript.text}", + style: AppConfig.messageTextStyle( + widget.messageEvent.event, + widget.textColor, + ).copyWith( + fontStyle: FontStyle.italic, ), ); } diff --git a/lib/pangea/toolbar/widgets/overlay_message.dart b/lib/pangea/toolbar/widgets/overlay_message.dart index a52f6f09c..e1f61ebfc 100644 --- a/lib/pangea/toolbar/widgets/overlay_message.dart +++ b/lib/pangea/toolbar/widgets/overlay_message.dart @@ -11,6 +11,7 @@ import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dar import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart'; import 'package:fluffychat/pangea/toolbar/enums/reading_assistance_mode_enum.dart'; import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart'; +import 'package:fluffychat/pangea/toolbar/widgets/message_speech_to_text_card.dart'; import 'package:fluffychat/utils/date_time_extension.dart'; import 'package:fluffychat/widgets/matrix.dart'; @@ -127,6 +128,8 @@ class OverlayMessage extends StatelessWidget { final showTranslation = overlayController.showTranslation && overlayController.translationText != null; + final showTranscription = pangeaMessageEvent?.isAudioMessage == true; + final content = Container( decoration: BoxDecoration( borderRadius: BorderRadius.circular( @@ -276,7 +279,7 @@ class OverlayMessage extends StatelessWidget { }, ) : content, - if (showTranslation) + if (showTranscription || showTranslation) SizedBox( width: messageWidth, child: Padding( @@ -286,13 +289,20 @@ class OverlayMessage extends StatelessWidget { 12.0, 12.0, ), - child: Text( - overlayController.translationText!, - style: - AppConfig.messageTextStyle(event, textColor).copyWith( - fontStyle: FontStyle.italic, - ), - ), + child: showTranscription + ? MessageSpeechToTextCard( + messageEvent: pangeaMessageEvent!, + textColor: textColor, + ) + : Text( + overlayController.translationText!, + style: AppConfig.messageTextStyle( + event, + textColor, + ).copyWith( + fontStyle: FontStyle.italic, + ), + ), ), ), ],