From e316517c1ed76269aaf925d6d68648723aedcf66 Mon Sep 17 00:00:00 2001 From: ggurdin <46800240+ggurdin@users.noreply.github.com> Date: Wed, 31 Dec 2025 10:30:42 -0500 Subject: [PATCH] fix: return bot local stt, ensure stt rep exists in request stt translation function (#5003) --- .../event_wrappers/pangea_message_event.dart | 125 +++++++++++------- .../pangea_representation_event.dart | 43 +----- 2 files changed, 81 insertions(+), 87 deletions(-) diff --git a/lib/pangea/events/event_wrappers/pangea_message_event.dart b/lib/pangea/events/event_wrappers/pangea_message_event.dart index b9839aa17..441fc4d54 100644 --- a/lib/pangea/events/event_wrappers/pangea_message_event.dart +++ b/lib/pangea/events/event_wrappers/pangea_message_event.dart @@ -13,6 +13,7 @@ import 'package:fluffychat/pangea/common/constants/model_keys.dart'; import 'package:fluffychat/pangea/events/event_wrappers/pangea_representation_event.dart'; import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart'; import 'package:fluffychat/pangea/events/models/representation_content_model.dart'; +import 'package:fluffychat/pangea/events/models/stt_translation_model.dart'; import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart'; import 'package:fluffychat/pangea/events/repo/language_detection_repo.dart'; import 'package:fluffychat/pangea/events/repo/language_detection_request.dart'; @@ -318,35 +319,23 @@ class PangeaMessageEvent { return null; } - SpeechToTextResponseModel? getSpeechToTextLocal() { - final rep = representations - .firstWhereOrNull( - (element) => element.content.speechToText != null, - ) - ?.content - .speechToText; + RepresentationEvent? _getSpeechToTextRepresentation() => + representations.firstWhereOrNull( + (element) => element.content.speechToText != null, + ); - if (rep != null) { - return rep; - } + SpeechToTextResponseModel? getSpeechToTextLocal() { + final rep = _getSpeechToTextRepresentation()?.content.speechToText; + if (rep != null) return rep; final rawBotTranscription = event.content.tryGetMap(ModelKey.botTranscription); if (rawBotTranscription != null) { try { - final stt = SpeechToTextResponseModel.fromJson( + return SpeechToTextResponseModel.fromJson( Map.from(rawBotTranscription), ); - _sendRepresentationEvent( - PangeaRepresentation( - langCode: stt.langCode, - text: stt.transcript.text, - originalSent: false, - originalWritten: false, - speechToText: stt, - ), - ); } catch (err, s) { ErrorHandler.logError( e: err, @@ -438,8 +427,9 @@ class PangeaMessageEvent { Future requestSpeechToText( String l1Code, - String l2Code, - ) async { + String l2Code, { + bool sendEvent = true, + }) async { if (!isAudioMessage) { throw 'Calling getSpeechToText on non-audio message'; } @@ -470,44 +460,87 @@ class PangeaMessageEvent { ); } - _representations = null; - _sendRepresentationEvent( - PangeaRepresentation( - langCode: result.result!.langCode, - text: result.result!.transcript.text, - originalSent: false, - originalWritten: false, - speechToText: result.result!, - ), - ); + if (sendEvent) { + _sendSttRepresentationEvent(result.result!); + } return result.result!; } + Future _sendSttRepresentationEvent( + SpeechToTextResponseModel stt, + ) async { + final representation = PangeaRepresentation( + langCode: stt.langCode, + text: stt.transcript.text, + originalSent: false, + originalWritten: false, + speechToText: stt, + ); + + _representations = null; + return room.sendPangeaEvent( + content: representation.toJson(), + parentEventId: eventId, + type: PangeaEventTypes.representation, + ); + } + Future requestSttTranslation({ required String langCode, required String l1Code, required String l2Code, }) async { - if (!representations.any( - (element) => element.content.speechToText != null, - )) { - await requestSpeechToText(l1Code, l2Code); - } - - final rep = representations.firstWhereOrNull( - (element) => element.content.speechToText != null, - ); + // First try to access the local translation event via a representation event + RepresentationEvent? rep = _getSpeechToTextRepresentation(); + final local = rep?.getSpeechToTextTranslationLocal(langCode); + if (local != null) return local.translation; + // The translation event needs a parent representation to relate to, + // so if the rep is null, we send a new representation event first. + // This happens mostly for bot audio messages, which store their transcripts + // in the original message event content. + SpeechToTextResponseModel? stt = rep?.content.speechToText; if (rep == null) { - throw Exception("No speech to text representation found"); + stt ??= await requestSpeechToText(l1Code, l2Code, sendEvent: false); + final repEvent = await _sendSttRepresentationEvent(stt); + if (repEvent == null) { + throw Exception("Failed to send representation event for STT"); + } + + rep = _getSpeechToTextRepresentation(); + if (rep == null) { + throw Exception("Failed to get representation event for STT"); + } } - final resp = await rep.requestSttTranslation( - userL1: l1Code, - userL2: l2Code, + // Make the translation request + final res = await FullTextTranslationRepo.get( + MatrixState.pangeaController.userController.accessToken, + FullTextTranslationRequestModel( + text: stt!.transcript.text, + tgtLang: l1Code, + userL2: l2Code, + userL1: l1Code, + ), ); - return resp.translation; + if (res.isError) { + throw res.error!; + } + + final translation = SttTranslationModel( + translation: res.result!, + langCode: l1Code, + ); + + // Send the translation event if the representation event exists + rep.event?.room.sendPangeaEvent( + content: translation.toJson(), + parentEventId: rep.event!.eventId, + type: PangeaEventTypes.sttTranslation, + ); + + return translation.translation; } Future requestRepresentationByDetectedLanguage() async { diff --git a/lib/pangea/events/event_wrappers/pangea_representation_event.dart b/lib/pangea/events/event_wrappers/pangea_representation_event.dart index 119016878..0dac12547 100644 --- a/lib/pangea/events/event_wrappers/pangea_representation_event.dart +++ b/lib/pangea/events/event_wrappers/pangea_representation_event.dart @@ -24,8 +24,6 @@ import 'package:fluffychat/pangea/events/repo/token_api_models.dart'; import 'package:fluffychat/pangea/events/repo/tokens_repo.dart'; import 'package:fluffychat/pangea/extensions/pangea_room_extension.dart'; import 'package:fluffychat/pangea/languages/language_constants.dart'; -import 'package:fluffychat/pangea/translation/full_text_translation_repo.dart'; -import 'package:fluffychat/pangea/translation/full_text_translation_request_model.dart'; import 'package:fluffychat/widgets/future_loading_dialog.dart'; import 'package:fluffychat/widgets/matrix.dart'; @@ -252,44 +250,7 @@ class RepresentationEvent { : Result.value(res.result!.tokens); } - Future requestSttTranslation({ - required String userL1, - required String userL2, - }) async { - if (content.speechToText == null) { - throw Exception( - "RepresentationEvent.getSttTranslation called on a representation without speechToText", - ); - } - - final local = sttTranslations.firstWhereOrNull((t) => t.langCode == userL1); - if (local != null) return local; - - final res = await FullTextTranslationRepo.get( - MatrixState.pangeaController.userController.accessToken, - FullTextTranslationRequestModel( - text: content.speechToText!.transcript.text, - tgtLang: userL1, - userL2: userL2, - userL1: userL1, - ), - ); - - if (res.isError) { - throw res.error!; - } - - final translation = SttTranslationModel( - translation: res.result!, - langCode: userL1, - ); - - _event?.room.sendPangeaEvent( - content: translation.toJson(), - parentEventId: _event!.eventId, - type: PangeaEventTypes.sttTranslation, - ); - - return translation; + SttTranslationModel? getSpeechToTextTranslationLocal(String langCode) { + return sttTranslations.firstWhereOrNull((t) => t.langCode == langCode); } }