From 296ddef06d60d43d29b4cc184a0cb0d9053c163d Mon Sep 17 00:00:00 2001 From: ggurdin Date: Wed, 11 Jun 2025 12:12:26 -0400 Subject: [PATCH] chore: add SST translation event --- .../events/constants/pangea_event_types.dart | 1 + .../controllers/message_data_controller.dart | 52 ++++++ .../event_wrappers/pangea_message_event.dart | 47 +++--- .../pangea_representation_event.dart | 67 ++++++++ .../event_wrappers/pangea_tokens_event.dart | 42 ----- .../events/models/stt_translation_model.dart | 23 +++ .../widgets/message_selection_overlay.dart | 59 ++++--- .../toolbar/widgets/overlay_message.dart | 16 +- .../toolbar/widgets/select_mode_buttons.dart | 158 ++++++++++++------ 9 files changed, 321 insertions(+), 144 deletions(-) delete mode 100644 lib/pangea/events/event_wrappers/pangea_tokens_event.dart create mode 100644 lib/pangea/events/models/stt_translation_model.dart diff --git a/lib/pangea/events/constants/pangea_event_types.dart b/lib/pangea/events/constants/pangea_event_types.dart index ba7097f57..fe2a47e18 100644 --- a/lib/pangea/events/constants/pangea_event_types.dart +++ b/lib/pangea/events/constants/pangea_event_types.dart @@ -16,6 +16,7 @@ class PangeaEventTypes { static const tokens = "pangea.tokens"; static const choreoRecord = "pangea.record"; static const representation = "pangea.representation"; + static const sttTranslation = "pangea.stt_translation"; // static const vocab = "p.vocab"; static const roomInfo = "pangea.roomtopic"; diff --git a/lib/pangea/events/controllers/message_data_controller.dart b/lib/pangea/events/controllers/message_data_controller.dart index bae9f2512..6bbcc5af2 100644 --- a/lib/pangea/events/controllers/message_data_controller.dart +++ b/lib/pangea/events/controllers/message_data_controller.dart @@ -13,6 +13,7 @@ import 'package:fluffychat/pangea/common/utils/error_handler.dart'; import 'package:fluffychat/pangea/events/constants/pangea_event_types.dart'; import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart'; import 'package:fluffychat/pangea/events/models/representation_content_model.dart'; +import 'package:fluffychat/pangea/events/models/stt_translation_model.dart'; import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart'; import 'package:fluffychat/pangea/events/repo/token_api_models.dart'; import 'package:fluffychat/pangea/extensions/pangea_room_extension.dart'; @@ -24,6 +25,7 @@ class MessageDataController extends BaseController { final Map> _tokensCache = {}; final Map> _representationCache = {}; + final Map> _sttTranslationCache = {}; late Timer _cacheTimer; MessageDataController(PangeaController pangeaController) { @@ -42,6 +44,7 @@ class MessageDataController extends BaseController { void _clearCache() { _tokensCache.clear(); _representationCache.clear(); + _sttTranslationCache.clear(); debugPrint("message data cache cleared."); } @@ -219,4 +222,53 @@ class MessageDataController extends BaseController { ); } } + + Future getSttTranslation({ + required String? repEventId, + required FullTextTranslationRequestModel req, + required Room? room, + }) => + _sttTranslationCache[req.hashCode] ??= _getSttTranslation( + repEventId: repEventId, + req: req, + room: room, + ).catchError((e, s) { + _sttTranslationCache.remove(req.hashCode); + return Future.error(e, s); + }); + + Future _getSttTranslation({ + required String? repEventId, + required FullTextTranslationRequestModel req, + required Room? room, + }) async { + final res = await FullTextTranslationRepo.translate( + accessToken: _pangeaController.userController.accessToken, + request: req, + ); + + final translation = SttTranslationModel( + translation: res.bestTranslation, + langCode: req.tgtLang, + ); + + if (repEventId != null && room != null) { + room + .sendPangeaEvent( + content: translation.toJson(), + parentEventId: repEventId, + type: PangeaEventTypes.sttTranslation, + ) + .catchError( + (e) => ErrorHandler.logError( + m: "error in _getSttTranslation.sendPangeaEvent", + e: e, + s: StackTrace.current, + data: req.toJson(), + ), + ); + } + + return translation; + } } diff --git a/lib/pangea/events/event_wrappers/pangea_message_event.dart b/lib/pangea/events/event_wrappers/pangea_message_event.dart index 1cf52e70f..d76519f38 100644 --- a/lib/pangea/events/event_wrappers/pangea_message_event.dart +++ b/lib/pangea/events/event_wrappers/pangea_message_event.dart @@ -16,6 +16,7 @@ import 'package:fluffychat/pangea/events/event_wrappers/pangea_representation_ev import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart'; import 'package:fluffychat/pangea/events/models/pangea_token_model.dart'; import 'package:fluffychat/pangea/events/models/representation_content_model.dart'; +import 'package:fluffychat/pangea/events/models/stt_translation_model.dart'; import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart'; import 'package:fluffychat/pangea/morphs/morph_features_enum.dart'; import 'package:fluffychat/pangea/practice_activities/activity_type_enum.dart'; @@ -281,32 +282,11 @@ class PangeaMessageEvent { ?.content .speechToText; - if (speechToTextLocal != null) return speechToTextLocal; + if (speechToTextLocal != null) { + return speechToTextLocal; + } final matrixFile = await _event.downloadAndDecryptAttachment(); - // Pangea# - // File? file; - - // TODO: Test on mobile and see if we need this case, doeesn't seem so - // if (!kIsWeb) { - // final tempDir = await getTemporaryDirectory(); - // final fileName = Uri.encodeComponent( - // // #Pangea - // // widget.event.attachmentOrThumbnailMxcUrl()!.pathSegments.last, - // widget.messageEvent.event - // .attachmentOrThumbnailMxcUrl()! - // .pathSegments - // .last, - // // Pangea# - // ); - // file = File('${tempDir.path}/${fileName}_${matrixFile.name}'); - // await file.writeAsBytes(matrixFile.bytes); - // } - - // audioFile = file; - - debugPrint("mimeType ${matrixFile.mimeType}"); - debugPrint("encoding ${mimeTypeToAudioEncoding(matrixFile.mimeType)}"); final SpeechToTextModel response = await MatrixState.pangeaController.speechToText.get( @@ -341,6 +321,25 @@ class PangeaMessageEvent { return response; } + Future sttTranslationByLanguageGlobal({ + required String langCode, + required String l1Code, + required String l2Code, + }) async { + if (!representations.any( + (element) => element.content.speechToText != null, + )) { + await getSpeechToText(l1Code, l2Code); + } + + final rep = representations.firstWhereOrNull( + (element) => element.content.speechToText != null, + ); + + if (rep == null) return null; + return rep.getSttTranslation(userL1: l1Code, userL2: l2Code); + } + PangeaMessageTokens? _tokensSafe(Map? content) { try { if (content == null) return null; diff --git a/lib/pangea/events/event_wrappers/pangea_representation_event.dart b/lib/pangea/events/event_wrappers/pangea_representation_event.dart index b7607e36e..12b41b83d 100644 --- a/lib/pangea/events/event_wrappers/pangea_representation_event.dart +++ b/lib/pangea/events/event_wrappers/pangea_representation_event.dart @@ -12,11 +12,13 @@ import 'package:sentry_flutter/sentry_flutter.dart'; import 'package:fluffychat/pangea/choreographer/event_wrappers/pangea_choreo_event.dart'; import 'package:fluffychat/pangea/choreographer/models/choreo_record.dart'; import 'package:fluffychat/pangea/choreographer/models/language_detection_model.dart'; +import 'package:fluffychat/pangea/choreographer/repo/full_text_translation_repo.dart'; import 'package:fluffychat/pangea/common/utils/error_handler.dart'; import 'package:fluffychat/pangea/events/constants/pangea_event_types.dart'; import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart'; import 'package:fluffychat/pangea/events/models/pangea_token_model.dart'; import 'package:fluffychat/pangea/events/models/representation_content_model.dart'; +import 'package:fluffychat/pangea/events/models/stt_translation_model.dart'; import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart'; import 'package:fluffychat/pangea/events/repo/token_api_models.dart'; import 'package:fluffychat/pangea/learning_settings/constants/language_constants.dart'; @@ -210,6 +212,71 @@ class RepresentationEvent { ); } + List get sttTranslations { + if (content.speechToText == null) return []; + if (_event == null) { + Sentry.addBreadcrumb( + Breadcrumb( + message: "_event and _sttTranslations both null", + ), + ); + return []; + } + + final Set sttEvents = _event!.aggregatedEvents( + timeline, + PangeaEventTypes.sttTranslation, + ); + + if (sttEvents.isEmpty) return []; + final List sttTranslations = []; + for (final event in sttEvents) { + try { + sttTranslations.add( + SttTranslationModel.fromJson(event.content), + ); + } catch (e) { + Sentry.addBreadcrumb( + Breadcrumb( + message: "Failed to parse STT translation", + data: { + "eventID": event.eventId, + "content": event.content, + "error": e.toString(), + }, + ), + ); + } + } + + return sttTranslations; + } + + Future getSttTranslation({ + required String userL1, + required String userL2, + }) async { + if (content.speechToText == null) { + throw Exception( + "RepresentationEvent.getSttTranslation called on a representation without speechToText", + ); + } + + final local = sttTranslations.firstWhereOrNull((t) => t.langCode == userL1); + if (local != null) return local; + + return MatrixState.pangeaController.messageData.getSttTranslation( + repEventId: _event?.eventId, + room: _event?.room, + req: FullTextTranslationRequestModel( + text: content.speechToText!.transcript.text, + tgtLang: userL1, + userL2: userL2, + userL1: userL1, + ), + ); + } + ChoreoRecord? get choreo { if (_choreo != null) return _choreo; diff --git a/lib/pangea/events/event_wrappers/pangea_tokens_event.dart b/lib/pangea/events/event_wrappers/pangea_tokens_event.dart deleted file mode 100644 index 126d9cff8..000000000 --- a/lib/pangea/events/event_wrappers/pangea_tokens_event.dart +++ /dev/null @@ -1,42 +0,0 @@ -import 'dart:developer'; - -import 'package:flutter/foundation.dart'; - -import 'package:matrix/matrix.dart'; - -import 'package:fluffychat/pangea/common/utils/error_handler.dart'; -import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart'; -import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart'; -import '../constants/pangea_event_types.dart'; - -class TokensEvent { - Event event; - PangeaMessageTokens? _content; - - TokensEvent({required this.event}) { - if (event.type != PangeaEventTypes.tokens) { - throw Exception( - "${event.type} should not be used to make a TokensEvent", - ); - } - } - - PangeaMessageTokens? get _pangeaMessageTokens { - try { - _content ??= event.getPangeaContent(); - return _content!; - } catch (err, s) { - debugger(when: kDebugMode); - ErrorHandler.logError( - e: err, - s: s, - data: { - "event": event.toJson(), - }, - ); - return null; - } - } - - PangeaMessageTokens? get tokens => _pangeaMessageTokens; -} diff --git a/lib/pangea/events/models/stt_translation_model.dart b/lib/pangea/events/models/stt_translation_model.dart new file mode 100644 index 000000000..47f31be4f --- /dev/null +++ b/lib/pangea/events/models/stt_translation_model.dart @@ -0,0 +1,23 @@ +class SttTranslationModel { + final String translation; + final String langCode; + + SttTranslationModel({ + required this.translation, + required this.langCode, + }); + + factory SttTranslationModel.fromJson(Map json) { + return SttTranslationModel( + translation: json['translation'] as String, + langCode: json['lang_code'] as String, + ); + } + + Map toJson() { + return { + 'translation': translation, + 'lang_code': langCode, + }; + } +} diff --git a/lib/pangea/toolbar/widgets/message_selection_overlay.dart b/lib/pangea/toolbar/widgets/message_selection_overlay.dart index 73451081f..f2492126d 100644 --- a/lib/pangea/toolbar/widgets/message_selection_overlay.dart +++ b/lib/pangea/toolbar/widgets/message_selection_overlay.dart @@ -30,6 +30,7 @@ import 'package:fluffychat/pangea/toolbar/controllers/text_to_speech_controller. import 'package:fluffychat/pangea/toolbar/controllers/tts_controller.dart'; import 'package:fluffychat/pangea/toolbar/enums/message_mode_enum.dart'; import 'package:fluffychat/pangea/toolbar/enums/reading_assistance_mode_enum.dart'; +import 'package:fluffychat/pangea/toolbar/models/speech_to_text_models.dart'; import 'package:fluffychat/pangea/toolbar/reading_assistance_input_row/morph_selection.dart'; import 'package:fluffychat/pangea/toolbar/widgets/message_selection_positioner.dart'; import 'package:fluffychat/pangea/toolbar/widgets/reading_assistance_content.dart'; @@ -91,12 +92,15 @@ class MessageOverlayController extends State ReadingAssistanceMode? readingAssistanceMode; // default mode - bool showTranslation = false; - String? translationText; - - String? transcriptionText; + SpeechToTextModel? transcription; String? transcriptionError; + bool showTranslation = false; + String? translation; + + bool showSpeechTranslation = false; + String? speechTranslation; + double maxWidth = AppConfig.toolbarMinWidth; ///////////////////////////////////// @@ -574,33 +578,50 @@ class MessageOverlayController extends State ); } - void setShowTranslation(bool show, String? translation) { - if (showTranslation == show) return; - if (show && translation == null) return; - + void setTranslation(String value) { if (mounted) { - setState(() { - showTranslation = show; - translationText = show ? translation : null; - }); + setState(() => translation = value); } } - void setTranscriptionText(String transcription) { + void setShowTranslation(bool show) { + if (!mounted) return; + if (translation == null) { + setState(() => showTranslation = false); + } + + if (showTranslation == show) return; + setState(() => showTranslation = show); + } + + void setSpeechTranslation(String value) { + if (mounted) { + setState(() => speechTranslation = value); + } + } + + void setShowSpeechTranslation(bool show) { + if (!mounted) return; + if (speechTranslation == null) { + setState(() => showSpeechTranslation = false); + } + + if (showSpeechTranslation == show) return; + setState(() => showSpeechTranslation = show); + } + + void setTranscription(SpeechToTextModel value) { if (mounted) { setState(() { transcriptionError = null; - transcriptionText = transcription; + transcription = value; }); } } - void setTranscriptionError(String error) { + void setTranscriptionError(String value) { if (mounted) { - setState(() { - transcriptionText = null; - transcriptionError = error; - }); + setState(() => transcriptionError = value); } } diff --git a/lib/pangea/toolbar/widgets/overlay_message.dart b/lib/pangea/toolbar/widgets/overlay_message.dart index c4e848851..91a528738 100644 --- a/lib/pangea/toolbar/widgets/overlay_message.dart +++ b/lib/pangea/toolbar/widgets/overlay_message.dart @@ -135,10 +135,13 @@ class OverlayMessage extends StatelessWidget { event.numberEmotes <= 3); final showTranslation = overlayController.showTranslation && - overlayController.translationText != null; + overlayController.translation != null; final showTranscription = pangeaMessageEvent?.isAudioMessage == true; + final showSpeechTranslation = overlayController.showSpeechTranslation && + overlayController.speechTranslation != null; + final content = Container( decoration: BoxDecoration( borderRadius: BorderRadius.circular( @@ -296,10 +299,11 @@ class OverlayMessage extends StatelessWidget { ), ], ) - : overlayController.transcriptionText != null + : overlayController.transcription != null ? SingleChildScrollView( child: Text( - overlayController.transcriptionText!, + overlayController + .transcription!.transcript.text, style: AppConfig.messageTextStyle( event, textColor, @@ -323,7 +327,7 @@ class OverlayMessage extends StatelessWidget { }, ) : content, - if (showTranslation) + if (showTranslation || showSpeechTranslation) Container( width: messageWidth, constraints: const BoxConstraints( @@ -338,7 +342,9 @@ class OverlayMessage extends StatelessWidget { ), child: SingleChildScrollView( child: Text( - overlayController.translationText!, + showTranslation + ? overlayController.translation! + : overlayController.speechTranslation!, style: AppConfig.messageTextStyle( event, textColor, diff --git a/lib/pangea/toolbar/widgets/select_mode_buttons.dart b/lib/pangea/toolbar/widgets/select_mode_buttons.dart index 2b434a835..76cca02f3 100644 --- a/lib/pangea/toolbar/widgets/select_mode_buttons.dart +++ b/lib/pangea/toolbar/widgets/select_mode_buttons.dart @@ -18,7 +18,6 @@ import 'package:fluffychat/pangea/common/widgets/pressable_button.dart'; import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart'; import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart'; import 'package:fluffychat/pangea/events/models/representation_content_model.dart'; -import 'package:fluffychat/pangea/toolbar/models/speech_to_text_models.dart'; import 'package:fluffychat/pangea/toolbar/widgets/message_audio_card.dart'; import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart'; import 'package:fluffychat/widgets/matrix.dart'; @@ -88,10 +87,12 @@ class SelectModeButtonsState extends State { StreamSubscription? _onAudioPositionChanged; bool _isLoadingTranslation = false; - PangeaRepresentation? _repEvent; String? _translationError; - SpeechToTextModel? _speechToTextResponse; + bool _isLoadingSpeechTranslation = false; + String? _speechTranslationError; + + Completer? _transcriptionCompleter; @override void initState() { @@ -113,7 +114,7 @@ class SelectModeButtonsState extends State { }); if (messageEvent?.isAudioMessage == true) { - _loadTranscription(); + _fetchTranscription(); } } @@ -129,9 +130,9 @@ class SelectModeButtonsState extends State { widget.overlayController.pangeaMessageEvent; String? get l1Code => - MatrixState.pangeaController.languageController.activeL1Code(); + MatrixState.pangeaController.languageController.userL1?.langCodeShort; String? get l2Code => - MatrixState.pangeaController.languageController.activeL2Code(); + MatrixState.pangeaController.languageController.userL2?.langCodeShort; void _clear() { setState(() { @@ -140,10 +141,8 @@ class SelectModeButtonsState extends State { }); widget.overlayController.updateSelectedSpan(null); - - if (_selectedMode == SelectMode.translate) { - widget.overlayController.setShowTranslation(false, null); - } + widget.overlayController.setShowTranslation(false); + widget.overlayController.setShowSpeechTranslation(false); } Future _updateMode(SelectMode? mode) async { @@ -177,7 +176,13 @@ class SelectModeButtonsState extends State { } if (_selectedMode == SelectMode.translate) { - await _loadTranslation(); + await _fetchTranslation(); + widget.overlayController.setShowTranslation(true); + } + + if (_selectedMode == SelectMode.speechTranslation) { + await _fetchSpeechTranslation(); + widget.overlayController.setShowSpeechTranslation(true); } } @@ -265,67 +270,68 @@ class SelectModeButtonsState extends State { } } - Future _fetchRepresentation() async { - if (l1Code == null || messageEvent == null || _repEvent != null) { + Future _fetchTranslation() async { + if (l1Code == null || + messageEvent == null || + widget.overlayController.translation != null) { return; } - _repEvent = messageEvent!.representationByLanguage(l1Code!)?.content; - if (_repEvent == null && mounted) { - _repEvent = await messageEvent?.representationByLanguageGlobal( + try { + if (mounted) setState(() => _isLoadingTranslation = true); + + PangeaRepresentation? rep = + messageEvent!.representationByLanguage(l1Code!)?.content; + + rep ??= await messageEvent?.representationByLanguageGlobal( langCode: l1Code!, ); + + widget.overlayController.setTranslation(rep!.text); + } catch (e, s) { + _translationError = e.toString(); + ErrorHandler.logError( + e: e, + s: s, + m: 'Error fetching translation', + data: { + 'l1Code': l1Code, + 'messageEvent': messageEvent?.event.toJson(), + }, + ); + } finally { + if (mounted) setState(() => _isLoadingTranslation = false); } } Future _fetchTranscription() async { - if (l1Code == null || messageEvent == null || _repEvent != null) { - return; - } - - _speechToTextResponse ??= await messageEvent!.getSpeechToText( - l1Code!, - l2Code!, - ); - } - - Future _loadTranslation() async { - if (!mounted) return; - setState(() => _isLoadingTranslation = true); - try { - await _fetchRepresentation(); - if (_repEvent == null) { - throw "No representation found for the selected language."; + if (_transcriptionCompleter != null) { + // If a transcription is already in progress, wait for it to complete + await _transcriptionCompleter!.future; + return; } - widget.overlayController.setShowTranslation( - true, - _repEvent!.text, - ); - } catch (err) { - _translationError = err.toString(); - ErrorHandler.logError( - e: err, - data: {}, - ); - } + _transcriptionCompleter = Completer(); + if (l1Code == null || messageEvent == null) { + _transcriptionCompleter?.completeError( + 'Language code or message event is null', + ); + return; + } - if (mounted) { - setState(() => _isLoadingTranslation = false); - } - } - - Future _loadTranscription() async { - try { - await _fetchTranscription(); - widget.overlayController.setTranscriptionText( - _speechToTextResponse!.transcript.text, + final resp = await messageEvent!.getSpeechToText( + l1Code!, + l2Code!, ); + + widget.overlayController.setTranscription(resp!); + _transcriptionCompleter?.complete(resp.transcript.text); } catch (err) { widget.overlayController.setTranscriptionError( err.toString(), ); + _transcriptionCompleter?.completeError(err); ErrorHandler.logError( e: err, data: {}, @@ -333,12 +339,54 @@ class SelectModeButtonsState extends State { } } + Future _fetchSpeechTranslation() async { + if (messageEvent == null || + l1Code == null || + l2Code == null || + widget.overlayController.speechTranslation != null) { + return; + } + + try { + setState(() => _isLoadingSpeechTranslation = true); + + if (widget.overlayController.transcription == null) { + await _fetchTranscription(); + if (widget.overlayController.transcription == null) { + throw Exception('Transcription is null'); + } + } + + final translation = await messageEvent!.sttTranslationByLanguageGlobal( + langCode: l1Code!, + l1Code: l1Code!, + l2Code: l2Code!, + ); + if (translation == null) { + throw Exception('Translation is null'); + } + + widget.overlayController.setSpeechTranslation(translation.translation); + } catch (err, s) { + debugPrint("Error fetching speech translation: $err, $s"); + _speechTranslationError = err.toString(); + ErrorHandler.logError( + e: err, + data: {}, + ); + } finally { + if (mounted) setState(() => _isLoadingSpeechTranslation = false); + } + } + bool get _isError { switch (_selectedMode) { case SelectMode.audio: return _audioError != null; case SelectMode.translate: return _translationError != null; + case SelectMode.speechTranslation: + return _speechTranslationError != null; default: return false; } @@ -350,6 +398,8 @@ class SelectModeButtonsState extends State { return _isLoadingAudio; case SelectMode.translate: return _isLoadingTranslation; + case SelectMode.speechTranslation: + return _isLoadingSpeechTranslation; default: return false; }