From 4a7e9dade9390004ce77389f6443378158fae87e Mon Sep 17 00:00:00 2001 From: ggurdin Date: Tue, 10 Jun 2025 12:52:38 -0400 Subject: [PATCH] feat: add toolbar buttons for audio messages --- .../widgets/message_selection_overlay.dart | 16 ++ .../widgets/message_selection_positioner.dart | 20 ++- .../widgets/message_speech_to_text_card.dart | 119 ------------ .../toolbar/widgets/overlay_message.dart | 50 ++++-- .../toolbar/widgets/select_mode_buttons.dart | 170 +++++++++++++----- 5 files changed, 190 insertions(+), 185 deletions(-) delete mode 100644 lib/pangea/toolbar/widgets/message_speech_to_text_card.dart diff --git a/lib/pangea/toolbar/widgets/message_selection_overlay.dart b/lib/pangea/toolbar/widgets/message_selection_overlay.dart index 844eafc26..4087457f6 100644 --- a/lib/pangea/toolbar/widgets/message_selection_overlay.dart +++ b/lib/pangea/toolbar/widgets/message_selection_overlay.dart @@ -91,9 +91,13 @@ class MessageOverlayController extends State final GlobalKey wordZoomKey = GlobalKey(); ReadingAssistanceMode? readingAssistanceMode; // default mode + bool showTranslation = false; String? translationText; + bool showTranscription = false; + String? transcriptText; + double maxWidth = AppConfig.toolbarMinWidth; ///////////////////////////////////// @@ -589,6 +593,18 @@ class MessageOverlayController extends State } } + void setShowTranscription(bool show, String? transcription) { + if (showTranscription == show) return; + if (show && transcription == null) return; + + if (mounted) { + setState(() { + showTranscription = show; + transcriptText = show ? transcription : null; + }); + } + } + ///////////////////////////////////// /// Build ///////////////////////////////////// diff --git a/lib/pangea/toolbar/widgets/message_selection_positioner.dart b/lib/pangea/toolbar/widgets/message_selection_positioner.dart index 53a8436ac..555154ad8 100644 --- a/lib/pangea/toolbar/widgets/message_selection_positioner.dart +++ b/lib/pangea/toolbar/widgets/message_selection_positioner.dart @@ -490,10 +490,22 @@ class MessageSelectionPositionerState extends State // measurement for items in the toolbar - bool get _showButtons => - (widget.pangeaMessageEvent?.shouldShowToolbar ?? false) && - widget.pangeaMessageEvent?.event.messageType == MessageTypes.Text && - (widget.pangeaMessageEvent?.messageDisplayLangIsL2 ?? false); + bool get _showButtons { + if (!(widget.pangeaMessageEvent?.shouldShowToolbar ?? false)) { + return false; + } + + final type = widget.pangeaMessageEvent?.event.messageType; + if (![MessageTypes.Text, MessageTypes.Audio].contains(type)) { + return false; + } + + if (type == MessageTypes.Text) { + return widget.pangeaMessageEvent?.messageDisplayLangIsL2 ?? false; + } + + return true; + } bool get showPracticeButtons => _showButtons && diff --git a/lib/pangea/toolbar/widgets/message_speech_to_text_card.dart b/lib/pangea/toolbar/widgets/message_speech_to_text_card.dart deleted file mode 100644 index 39f06683d..000000000 --- a/lib/pangea/toolbar/widgets/message_speech_to_text_card.dart +++ /dev/null @@ -1,119 +0,0 @@ -import 'dart:developer'; - -import 'package:flutter/foundation.dart'; -import 'package:flutter/material.dart'; - -import 'package:fluffychat/config/app_config.dart'; -import 'package:fluffychat/l10n/l10n.dart'; -import 'package:fluffychat/pangea/common/utils/error_handler.dart'; -import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart'; -import 'package:fluffychat/pangea/toolbar/models/speech_to_text_models.dart'; -import 'package:fluffychat/widgets/matrix.dart'; - -class MessageSpeechToTextCard extends StatefulWidget { - final PangeaMessageEvent messageEvent; - final Color textColor; - - const MessageSpeechToTextCard({ - super.key, - required this.messageEvent, - required this.textColor, - }); - - @override - MessageSpeechToTextCardState createState() => MessageSpeechToTextCardState(); -} - -class MessageSpeechToTextCardState extends State { - SpeechToTextModel? _speechToTextResponse; - - bool _fetchingTranscription = true; - Object? error; - - String? get l1Code => - MatrixState.pangeaController.languageController.activeL1Code(); - String? get l2Code => - MatrixState.pangeaController.languageController.activeL2Code(); - - @override - void initState() { - super.initState(); - _fetchTranscription(); - } - - // look for transcription in message event - // if not found, call API to transcribe audio - Future _fetchTranscription() async { - try { - if (l1Code == null || l2Code == null) { - throw Exception('Language selection not found'); - } - - _speechToTextResponse ??= await widget.messageEvent.getSpeechToText( - l1Code!, - l2Code!, - ); - } catch (e, s) { - debugger(when: kDebugMode); - error = e; - ErrorHandler.logError( - e: e, - s: s, - data: widget.messageEvent.event.content, - ); - } finally { - if (mounted) { - setState(() => _fetchingTranscription = false); - } - } - } - - @override - Widget build(BuildContext context) { - if (_fetchingTranscription) { - return const LinearProgressIndicator(); - } - - // // done fetching but not results means some kind of error - if (_speechToTextResponse == null || error != null) { - return Row( - spacing: 8.0, - children: [ - Flexible( - child: RichText( - text: TextSpan( - style: AppConfig.messageTextStyle( - widget.messageEvent.event, - widget.textColor, - ), - children: [ - WidgetSpan( - alignment: PlaceholderAlignment.middle, - child: Icon( - Icons.error, - color: Theme.of(context).colorScheme.error, - ), - ), - const TextSpan(text: " "), - TextSpan( - text: L10n.of(context).oopsSomethingWentWrong, - ), - ], - ), - ), - ), - ], - ); - } - - return Text( - "${_speechToTextResponse?.transcript.text}", - style: AppConfig.messageTextStyle( - widget.messageEvent.event, - widget.textColor, - ).copyWith( - fontStyle: FontStyle.italic, - ), - ); - } -} diff --git a/lib/pangea/toolbar/widgets/overlay_message.dart b/lib/pangea/toolbar/widgets/overlay_message.dart index b5da23fc4..8b4ccb611 100644 --- a/lib/pangea/toolbar/widgets/overlay_message.dart +++ b/lib/pangea/toolbar/widgets/overlay_message.dart @@ -11,7 +11,6 @@ import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dar import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart'; import 'package:fluffychat/pangea/toolbar/enums/reading_assistance_mode_enum.dart'; import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart'; -import 'package:fluffychat/pangea/toolbar/widgets/message_speech_to_text_card.dart'; import 'package:fluffychat/utils/date_time_extension.dart'; import 'package:fluffychat/utils/file_description.dart'; import 'package:fluffychat/widgets/matrix.dart'; @@ -137,7 +136,8 @@ class OverlayMessage extends StatelessWidget { final showTranslation = overlayController.showTranslation && overlayController.translationText != null; - final showTranscription = pangeaMessageEvent?.isAudioMessage == true; + final showTranscription = overlayController.showTranscription && + overlayController.transcriptText != null; final content = Container( decoration: BoxDecoration( @@ -270,6 +270,27 @@ class OverlayMessage extends StatelessWidget { child: Column( crossAxisAlignment: CrossAxisAlignment.start, children: [ + if (showTranscription) + Container( + width: messageWidth, + constraints: const BoxConstraints( + maxHeight: AppConfig.audioTranscriptionMaxHeight, + ), + child: Padding( + padding: const EdgeInsets.all(12.0), + child: SingleChildScrollView( + child: Text( + overlayController.transcriptText!, + style: AppConfig.messageTextStyle( + event, + textColor, + ).copyWith( + fontStyle: FontStyle.italic, + ), + ), + ), + ), + ), sizeAnimation != null ? AnimatedBuilder( animation: sizeAnimation!, @@ -282,7 +303,7 @@ class OverlayMessage extends StatelessWidget { }, ) : content, - if (showTranscription || showTranslation) + if (showTranslation) Container( width: messageWidth, constraints: const BoxConstraints( @@ -296,20 +317,15 @@ class OverlayMessage extends StatelessWidget { 12.0, ), child: SingleChildScrollView( - child: showTranscription - ? MessageSpeechToTextCard( - messageEvent: pangeaMessageEvent!, - textColor: textColor, - ) - : Text( - overlayController.translationText!, - style: AppConfig.messageTextStyle( - event, - textColor, - ).copyWith( - fontStyle: FontStyle.italic, - ), - ), + child: Text( + overlayController.translationText!, + style: AppConfig.messageTextStyle( + event, + textColor, + ).copyWith( + fontStyle: FontStyle.italic, + ), + ), ), ), ), diff --git a/lib/pangea/toolbar/widgets/select_mode_buttons.dart b/lib/pangea/toolbar/widgets/select_mode_buttons.dart index 9a5a4b700..ea93660c7 100644 --- a/lib/pangea/toolbar/widgets/select_mode_buttons.dart +++ b/lib/pangea/toolbar/widgets/select_mode_buttons.dart @@ -18,6 +18,7 @@ import 'package:fluffychat/pangea/common/widgets/pressable_button.dart'; import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart'; import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart'; import 'package:fluffychat/pangea/events/models/representation_content_model.dart'; +import 'package:fluffychat/pangea/toolbar/models/speech_to_text_models.dart'; import 'package:fluffychat/pangea/toolbar/widgets/message_audio_card.dart'; import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart'; import 'package:fluffychat/widgets/matrix.dart'; @@ -25,7 +26,8 @@ import 'package:fluffychat/widgets/matrix.dart'; enum SelectMode { audio(Icons.volume_up), translate(Icons.translate), - practice(Symbols.fitness_center); + practice(Symbols.fitness_center), + transcription(Icons.translate); final IconData icon; const SelectMode(this.icon); @@ -39,6 +41,8 @@ enum SelectMode { return l10n.translationTooltip; case SelectMode.practice: return l10n.practice; + case SelectMode.transcription: + return l10n.speechToTextTooltip; } } } @@ -61,6 +65,17 @@ class SelectModeButtonsState extends State { static const double iconWidth = 36.0; static const double buttonSize = 40.0; + static List get textModes => [ + SelectMode.audio, + SelectMode.translate, + SelectMode.practice, + ]; + + static List get audioModes => [ + SelectMode.transcription, + SelectMode.practice, + ]; + SelectMode? _selectedMode; final AudioPlayer _audioPlayer = AudioPlayer(); @@ -74,6 +89,11 @@ class SelectModeButtonsState extends State { bool _isLoadingTranslation = false; PangeaRepresentation? _repEvent; + String? _translationError; + + bool _isLoadingTranscription = false; + SpeechToTextModel? _speechToTextResponse; + String? _transcriptionError; @override void initState() { @@ -112,12 +132,21 @@ class SelectModeButtonsState extends State { MatrixState.pangeaController.languageController.activeL2Code(); void _clear() { - setState(() => _audioError = null); + setState(() { + _audioError = null; + _translationError = null; + _transcriptionError = null; + }); + widget.overlayController.updateSelectedSpan(null); if (_selectedMode == SelectMode.translate) { widget.overlayController.setShowTranslation(false, null); } + + if (_selectedMode == SelectMode.transcription) { + widget.overlayController.setShowTranscription(false, null); + } } Future _updateMode(SelectMode? mode) async { @@ -158,6 +187,15 @@ class SelectModeButtonsState extends State { _repEvent!.text, ); } + + if (_selectedMode == SelectMode.transcription) { + await _loadTranscription(); + if (_speechToTextResponse == null) return; + widget.overlayController.setShowTranscription( + true, + _speechToTextResponse!.transcript.text, + ); + } } Future _fetchAudio() async { @@ -257,6 +295,17 @@ class SelectModeButtonsState extends State { } } + Future _fetchTranscription() async { + if (l1Code == null || messageEvent == null || _repEvent != null) { + return; + } + + _speechToTextResponse ??= await messageEvent!.getSpeechToText( + l1Code!, + l2Code!, + ); + } + Future _loadTranslation() async { if (!mounted) return; setState(() => _isLoadingTranslation = true); @@ -264,6 +313,7 @@ class SelectModeButtonsState extends State { try { await _fetchRepresentation(); } catch (err) { + _translationError = err.toString(); ErrorHandler.logError( e: err, data: {}, @@ -275,50 +325,78 @@ class SelectModeButtonsState extends State { } } - Widget icon(SelectMode mode) { - if (mode == SelectMode.audio) { - if (_audioError != null) { - return Icon( - Icons.error_outline, - size: 20, - color: Theme.of(context).colorScheme.error, - ); - } - if (_isLoadingAudio) { - return const Center( - child: SizedBox( - height: 20.0, - width: 20.0, - child: CircularProgressIndicator.adaptive(), - ), - ); - } else { - return Icon( - _audioPlayer.playerState.playing == true - ? Icons.pause_outlined - : Icons.volume_up, - size: 20, - color: mode == _selectedMode ? Colors.white : null, - ); - } + Future _loadTranscription() async { + if (!mounted) return; + setState(() => _isLoadingTranscription = true); + + try { + await _fetchTranscription(); + } catch (err) { + _transcriptionError = err.toString(); + ErrorHandler.logError( + e: err, + data: {}, + ); } - if (mode == SelectMode.translate) { - if (_isLoadingTranslation) { - return const Center( - child: SizedBox( - height: 20.0, - width: 20.0, - child: CircularProgressIndicator.adaptive(), - ), - ); - } else if (_repEvent != null) { - return Icon( - mode.icon, - size: 20, - color: mode == _selectedMode ? Colors.white : null, - ); - } + if (mounted) { + setState(() => _isLoadingTranscription = false); + } + } + + bool get _isError { + switch (_selectedMode) { + case SelectMode.audio: + return _audioError != null; + case SelectMode.translate: + return _translationError != null; + case SelectMode.transcription: + return _transcriptionError != null; + default: + return false; + } + } + + bool get _isLoading { + switch (_selectedMode) { + case SelectMode.audio: + return _isLoadingAudio; + case SelectMode.translate: + return _isLoadingTranslation; + case SelectMode.transcription: + return _isLoadingTranscription; + default: + return false; + } + } + + Widget icon(SelectMode mode) { + if (_isError && mode == _selectedMode) { + return Icon( + Icons.error_outline, + size: 20, + color: Theme.of(context).colorScheme.error, + ); + } + + if (_isLoading && mode == _selectedMode) { + return const Center( + child: SizedBox( + height: 20.0, + width: 20.0, + child: CircularProgressIndicator.adaptive(), + ), + ); + } + + if (mode == SelectMode.audio) { + return Icon( + _audioPlayer.playerState.playing == true + ? Icons.pause_outlined + : Icons.volume_up, + size: 20, + color: mode == _selectedMode ? Colors.white : null, + ); } return Icon( @@ -330,6 +408,8 @@ class SelectModeButtonsState extends State { @override Widget build(BuildContext context) { + final modes = messageEvent?.isAudioMessage == true ? audioModes : textModes; + return Container( height: AppConfig.toolbarButtonsHeight, alignment: Alignment.bottomCenter, @@ -338,7 +418,7 @@ class SelectModeButtonsState extends State { mainAxisSize: MainAxisSize.min, spacing: 4.0, children: [ - for (final mode in SelectMode.values) + for (final mode in modes) Tooltip( message: mode.tooltip(context), child: PressableButton(