diff --git a/lib/pages/chat/chat.dart b/lib/pages/chat/chat.dart index ac3ca9fca..61d69dd0c 100644 --- a/lib/pages/chat/chat.dart +++ b/lib/pages/chat/chat.dart @@ -2130,7 +2130,7 @@ class ChatController extends State MatrixState.pangeaController.languageController.userL2?.langCodeShort ?? LanguageKeys.unknownLanguage, ); - if (stt == null || stt.transcript.sttTokens.isEmpty) return; + if (stt.transcript.sttTokens.isEmpty) return; final constructs = stt.constructs(roomId, eventId); if (constructs.isEmpty) return; diff --git a/lib/pangea/common/utils/async_state.dart b/lib/pangea/common/utils/async_state.dart index 91c4e36db..a9fd60cb5 100644 --- a/lib/pangea/common/utils/async_state.dart +++ b/lib/pangea/common/utils/async_state.dart @@ -1,3 +1,7 @@ +import 'package:flutter/material.dart'; + +import 'package:fluffychat/pangea/common/utils/error_handler.dart'; + /// A generic sealed class that represents the state of an asynchronous operation. sealed class AsyncState { /// Base constructor for all asynchronous state variants. @@ -53,3 +57,47 @@ class AsyncError extends AsyncState { /// Creates an error [AsyncState] with an [error]. const AsyncError(this.error); } + +abstract class AsyncLoader { + final ValueNotifier> state = ValueNotifier(AsyncState.idle()); + bool _disposed = false; + + bool get isIdle => state.value is AsyncIdle; + bool get isLoading => state.value is AsyncLoading; + bool get isLoaded => state.value is AsyncLoaded; + bool get isError => state.value is AsyncError; + + T? get value => isLoaded ? (state.value as AsyncLoaded).value : null; + + void dispose() { + _disposed = true; + state.dispose(); + } + + Future fetch(); + + Future load() async { + if (state.value is AsyncLoading || state.value is AsyncLoaded) { + // If already loading or loaded, do nothing. + return; + } + + state.value = AsyncState.loading(); + + try { + final result = await fetch(); + if (_disposed) return; + state.value = AsyncState.loaded(result); + } catch (e, s) { + ErrorHandler.logError( + e: e, + s: s, + data: {}, + ); + + if (!_disposed) { + state.value = AsyncState.error(e); + } + } + } +} diff --git a/lib/pangea/events/event_wrappers/pangea_message_event.dart b/lib/pangea/events/event_wrappers/pangea_message_event.dart index d75e26855..ef75245e4 100644 --- a/lib/pangea/events/event_wrappers/pangea_message_event.dart +++ b/lib/pangea/events/event_wrappers/pangea_message_event.dart @@ -13,7 +13,6 @@ import 'package:fluffychat/pangea/common/constants/model_keys.dart'; import 'package:fluffychat/pangea/events/controllers/message_data_controller.dart'; import 'package:fluffychat/pangea/events/event_wrappers/pangea_representation_event.dart'; import 'package:fluffychat/pangea/events/models/representation_content_model.dart'; -import 'package:fluffychat/pangea/events/models/stt_translation_model.dart'; import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart'; import 'package:fluffychat/pangea/events/repo/language_detection_repo.dart'; import 'package:fluffychat/pangea/events/repo/language_detection_request.dart'; @@ -258,44 +257,20 @@ class PangeaMessageEvent { .speechToText; } - Future getSpeechToText( + Future getSpeechToText( String l1Code, String l2Code, ) async { if (!isAudioMessage) { - ErrorHandler.logError( - e: 'Calling getSpeechToText on non-audio message', - s: StackTrace.current, - data: { - "content": _event.content, - "eventId": _event.eventId, - "roomId": _event.roomId, - "userId": _event.room.client.userID, - "account_data": _event.room.client.accountData, - }, - ); - return null; + throw 'Calling getSpeechToText on non-audio message'; } final rawBotTranscription = event.content.tryGetMap(ModelKey.botTranscription); if (rawBotTranscription != null) { - SpeechToTextModel botTranscription; - try { - botTranscription = SpeechToTextModel.fromJson( - Map.from(rawBotTranscription), - ); - } catch (err, s) { - ErrorHandler.logError( - e: err, - s: s, - data: { - "event": _event.toJson(), - }, - m: "error parsing botTranscription", - ); - return null; - } + final SpeechToTextModel botTranscription = SpeechToTextModel.fromJson( + Map.from(rawBotTranscription), + ); _representations ??= []; _representations!.add( @@ -361,7 +336,7 @@ class PangeaMessageEvent { return response; } - Future sttTranslationByLanguageGlobal({ + Future sttTranslationByLanguageGlobal({ required String langCode, required String l1Code, required String l2Code, @@ -376,8 +351,12 @@ class PangeaMessageEvent { (element) => element.content.speechToText != null, ); - if (rep == null) return null; - return rep.getSttTranslation(userL1: l1Code, userL2: l2Code); + if (rep == null) { + throw Exception("No speech to text representation found"); + } + + final resp = await rep.getSttTranslation(userL1: l1Code, userL2: l2Code); + return resp.translation; } PangeaMessageTokens? _tokensSafe(Map? content) { @@ -572,7 +551,7 @@ class PangeaMessageEvent { ); } - Future l1Respresentation() async { + Future l1Respresentation() async { if (l1Code == null || l2Code == null) { throw Exception("Missing language codes"); } @@ -595,7 +574,7 @@ class PangeaMessageEvent { ); } - if (rep != null) return rep.content; + if (rep != null) return rep.content.text; final String srcLang = includedIT ? (originalWritten?.langCode ?? l1Code!) @@ -603,7 +582,7 @@ class PangeaMessageEvent { // clear representations cache so the new representation event can be added when next requested _representations = null; - return MessageDataController.getPangeaRepresentation( + final resp = await MessageDataController.getPangeaRepresentation( req: FullTextTranslationRequestModel( text: includedIT ? originalWrittenContent : messageDisplayText, srcLang: srcLang, @@ -613,6 +592,7 @@ class PangeaMessageEvent { ), messageEvent: _event, ); + return resp.text; } RepresentationEvent? get originalSent => representations diff --git a/lib/pangea/toolbar/widgets/message_selection_overlay.dart b/lib/pangea/toolbar/widgets/message_selection_overlay.dart index e47b7eabd..9753d9718 100644 --- a/lib/pangea/toolbar/widgets/message_selection_overlay.dart +++ b/lib/pangea/toolbar/widgets/message_selection_overlay.dart @@ -134,16 +134,9 @@ class MessageOverlayController extends State } } - /// Decides whether an _initialSelectedToken should be used - /// for a first practice activity on the word meaning - Future _initializeSelectedToken() async { - // if there is no initial selected token, then we don't need to do anything - if (widget._initialSelectedToken == null) { - return; - } - - updateSelectedSpan(widget._initialSelectedToken!.text); - } + void _initializeSelectedToken() => widget._initialSelectedToken != null + ? updateSelectedSpan(widget._initialSelectedToken!.text) + : null; ///////////////////////////////////// /// State setting @@ -250,19 +243,6 @@ class MessageOverlayController extends State ?.firstWhereOrNull(isTokenSelected); } - bool get showLanguageAssistance { - if (!event.status.isSent || event.type != EventTypes.Message) { - return false; - } - - if (event.messageType == MessageTypes.Text) { - return pangeaMessageEvent.messageDisplayLangCode.split("-").first == - MatrixState.pangeaController.languageController.userL2!.langCodeShort; - } - - return event.messageType == MessageTypes.Audio; - } - /// If sentence TTS is playing a word, highlight that word in message overlay void highlightCurrentText(int currentPosition, List ttsTokens) { final List textToSelect = []; @@ -302,22 +282,8 @@ class MessageOverlayController extends State void onClickOverlayMessageToken( PangeaToken token, - ) { - // /// we don't want to associate the audio with the text in this mode - // if (practiceSelection?.hasActiveActivityByToken( - // ActivityTypeEnum.wordFocusListening, - // token, - // ) == - // false || - // !hideWordCardContent) { - // TtsController.tryToSpeak( - // token.text.content, - // targetID: null, - // langCode: pangeaMessageEvent.messageDisplayLangCode, - // ); - // } - updateSelectedSpan(token.text); - } + ) => + updateSelectedSpan(token.text); /// Whether the given token is currently selected or highlighted bool isTokenSelected(PangeaToken token) { diff --git a/lib/pangea/toolbar/widgets/over_message_overlay.dart b/lib/pangea/toolbar/widgets/over_message_overlay.dart index 8e92339f7..a75d9a6ab 100644 --- a/lib/pangea/toolbar/widgets/over_message_overlay.dart +++ b/lib/pangea/toolbar/widgets/over_message_overlay.dart @@ -58,7 +58,7 @@ class OverMessageOverlay extends StatelessWidget { ? controller.originalMessageSize.height : null, messageWidth: controller.widget.overlayController - .selectModeController.showingExtraContent + .selectModeController.isShowingExtraContent ? max(controller.originalMessageSize.width, 150) : controller.originalMessageSize.width, overlayController: controller.widget.overlayController, diff --git a/lib/pangea/toolbar/widgets/select_mode_buttons.dart b/lib/pangea/toolbar/widgets/select_mode_buttons.dart index 192a49cc3..5610ecb7b 100644 --- a/lib/pangea/toolbar/widgets/select_mode_buttons.dart +++ b/lib/pangea/toolbar/widgets/select_mode_buttons.dart @@ -146,17 +146,6 @@ class SelectModeButtonsState extends State { StreamSubscription? _playerStateSub; StreamSubscription? _audioSub; - static List get textModes => [ - SelectMode.audio, - SelectMode.translate, - SelectMode.practice, - SelectMode.emoji, - ]; - - static List get audioModes => [ - SelectMode.speechTranslation, - ]; - MatrixState? matrix; @override @@ -309,13 +298,7 @@ class SelectModeButtonsState extends State { @override Widget build(BuildContext context) { final theme = Theme.of(context); - final List modes = - widget.overlayController.showLanguageAssistance - ? messageEvent.isAudioMessage == true - ? audioModes - : textModes - : []; - + final modes = controller.readingAssistanceModes; return Material( type: MaterialType.transparency, child: SizedBox( diff --git a/lib/pangea/toolbar/widgets/select_mode_controller.dart b/lib/pangea/toolbar/widgets/select_mode_controller.dart index 9b49fe8b2..013294b52 100644 --- a/lib/pangea/toolbar/widgets/select_mode_controller.dart +++ b/lib/pangea/toolbar/widgets/select_mode_controller.dart @@ -7,7 +7,6 @@ import 'package:matrix/matrix.dart'; import 'package:path_provider/path_provider.dart'; import 'package:fluffychat/pangea/common/utils/async_state.dart'; -import 'package:fluffychat/pangea/common/utils/error_handler.dart'; import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart'; import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart'; import 'package:fluffychat/pangea/toolbar/models/speech_to_text_models.dart'; @@ -15,270 +14,173 @@ import 'package:fluffychat/pangea/toolbar/widgets/message_audio_card.dart'; import 'package:fluffychat/pangea/toolbar/widgets/select_mode_buttons.dart'; import 'package:fluffychat/widgets/matrix.dart'; +class _TranscriptionLoader extends AsyncLoader { + final PangeaMessageEvent messageEvent; + _TranscriptionLoader(this.messageEvent) : super(); + + @override + Future fetch() => messageEvent.getSpeechToText( + MatrixState.pangeaController.languageController.userL1!.langCodeShort, + MatrixState.pangeaController.languageController.userL2!.langCodeShort, + ); +} + +class _STTTranslationLoader extends AsyncLoader { + final PangeaMessageEvent messageEvent; + _STTTranslationLoader(this.messageEvent) : super(); + + @override + Future fetch() => messageEvent.sttTranslationByLanguageGlobal( + langCode: MatrixState + .pangeaController.languageController.userL1!.langCodeShort, + l1Code: MatrixState + .pangeaController.languageController.userL1!.langCodeShort, + l2Code: MatrixState + .pangeaController.languageController.userL2!.langCodeShort, + ); +} + +class _TranslationLoader extends AsyncLoader { + final PangeaMessageEvent messageEvent; + _TranslationLoader(this.messageEvent) : super(); + + @override + Future fetch() => messageEvent.l1Respresentation(); +} + +class _AudioLoader extends AsyncLoader<(PangeaAudioFile, File?)> { + final PangeaMessageEvent messageEvent; + _AudioLoader(this.messageEvent) : super(); + + @override + Future<(PangeaAudioFile, File?)> fetch() async { + final String langCode = messageEvent.messageDisplayLangCode; + + final Event? localEvent = messageEvent.getTextToSpeechLocal( + langCode, + messageEvent.messageDisplayText, + ); + + PangeaAudioFile? audioBytes; + if (localEvent != null) { + audioBytes = await localEvent.getPangeaAudioFile(); + } else { + audioBytes = await messageEvent.getMatrixAudioFile( + langCode, + ); + } + if (audioBytes == null) { + throw Exception('Audio bytes are null'); + } + + File? audioFile; + if (!kIsWeb) { + final tempDir = await getTemporaryDirectory(); + + File? file; + file = File('${tempDir.path}/${audioBytes.name}'); + await file.writeAsBytes(audioBytes.bytes); + audioFile = file; + } + + return (audioBytes, audioFile); + } +} + class SelectModeController { final PangeaMessageEvent messageEvent; + final _TranscriptionLoader _transcriptLoader; + final _TranslationLoader _translationLoader; + final _AudioLoader _audioLoader; + final _STTTranslationLoader _sttTranslationLoader; SelectModeController( this.messageEvent, - ); + ) : _transcriptLoader = _TranscriptionLoader(messageEvent), + _translationLoader = _TranslationLoader(messageEvent), + _audioLoader = _AudioLoader(messageEvent), + _sttTranslationLoader = _STTTranslationLoader(messageEvent); ValueNotifier selectedMode = ValueNotifier(null); - - final ValueNotifier> transcriptionState = - ValueNotifier>(const AsyncState.idle()); - - final ValueNotifier> translationState = - ValueNotifier>(const AsyncState.idle()); - - final ValueNotifier> speechTranslationState = - ValueNotifier>(const AsyncState.idle()); - - final ValueNotifier> audioState = - ValueNotifier>( - const AsyncState.idle(), - ); - final StreamController contentChangedStream = StreamController.broadcast(); - bool _disposed = false; - - bool get showingExtraContent => - (selectedMode.value == SelectMode.translate && - translationState.value is AsyncLoaded) || - (selectedMode.value == SelectMode.speechTranslation && - speechTranslationState.value is AsyncLoaded) || - transcriptionState.value is AsyncLoaded || - transcriptionState.value is AsyncError; - - String? get l1Code => - MatrixState.pangeaController.languageController.userL1?.langCodeShort; - String? get l2Code => - MatrixState.pangeaController.languageController.userL2?.langCodeShort; - - (PangeaAudioFile, File?)? get audioFile => audioState.value is AsyncLoaded - ? (audioState.value as AsyncLoaded<(PangeaAudioFile, File?)>).value - : null; - - ValueNotifier? modeStateNotifier(SelectMode mode) { - switch (mode) { - case SelectMode.audio: - return audioState; - case SelectMode.translate: - return translationState; - case SelectMode.speechTranslation: - return speechTranslationState; - default: - return null; - } - } - - ValueNotifier? get currentModeStateNotifier { - final mode = selectedMode.value; - if (mode == null) return null; - return modeStateNotifier(mode); - } - void dispose() { selectedMode.dispose(); - transcriptionState.dispose(); - translationState.dispose(); - speechTranslationState.dispose(); - audioState.dispose(); + _transcriptLoader.dispose(); + _translationLoader.dispose(); + _sttTranslationLoader.dispose(); + _audioLoader.dispose(); contentChangedStream.close(); - _disposed = true; } + static List get textModes => [ + SelectMode.audio, + SelectMode.translate, + SelectMode.practice, + SelectMode.emoji, + ]; + + static List get audioModes => [ + SelectMode.speechTranslation, + ]; + + ValueNotifier> get translationState => + _translationLoader.state; + + ValueNotifier> get transcriptionState => + _transcriptLoader.state; + + ValueNotifier> get speechTranslationState => + _sttTranslationLoader.state; + + (PangeaAudioFile, File?)? get audioFile => _audioLoader.value; + + List get readingAssistanceModes { + final validTypes = {MessageTypes.Text, MessageTypes.Audio}; + if (!messageEvent.event.status.isSent || + messageEvent.event.type != EventTypes.Message || + !validTypes.contains(messageEvent.event.messageType)) { + return []; + } + + if (messageEvent.event.messageType == MessageTypes.Text) { + final matchesL2 = messageEvent.messageDisplayLangCode.split("-").first == + MatrixState.pangeaController.languageController.userL2!.langCodeShort; + + return matchesL2 ? textModes : [SelectMode.translate]; + } + + return audioModes; + } + + bool get isLoading => currentModeStateNotifier?.value is AsyncLoading; + + bool get isShowingExtraContent => + (selectedMode.value == SelectMode.translate && + _translationLoader.isLoaded) || + (selectedMode.value == SelectMode.speechTranslation && + _sttTranslationLoader.isLoaded) || + _transcriptLoader.isLoaded || + _transcriptLoader.isError; + + ValueNotifier? get currentModeStateNotifier => + modeStateNotifier(selectedMode.value); + + ValueNotifier? modeStateNotifier(SelectMode? mode) => + switch (mode) { + SelectMode.audio => _audioLoader.state, + SelectMode.translate => _translationLoader.state, + SelectMode.speechTranslation => _sttTranslationLoader.state, + _ => null, + }; + void setSelectMode(SelectMode? mode) { if (selectedMode.value == mode) return; selectedMode.value = mode; } - Future fetchAudio() async { - audioState.value = const AsyncState.loading(); - try { - final String langCode = messageEvent.messageDisplayLangCode; - final Event? localEvent = messageEvent.getTextToSpeechLocal( - langCode, - messageEvent.messageDisplayText, - ); - - PangeaAudioFile? audioBytes; - if (localEvent != null) { - audioBytes = await localEvent.getPangeaAudioFile(); - } else { - audioBytes = await messageEvent.getMatrixAudioFile( - langCode, - ); - } - if (_disposed) return; - if (audioBytes == null) { - throw Exception('Audio bytes are null'); - } - - File? audioFile; - if (!kIsWeb) { - final tempDir = await getTemporaryDirectory(); - - File? file; - file = File('${tempDir.path}/${audioBytes.name}'); - await file.writeAsBytes(audioBytes.bytes); - audioFile = file; - } - - audioState.value = AsyncState.loaded((audioBytes, audioFile)); - } catch (e, s) { - ErrorHandler.logError( - e: e, - s: s, - m: 'something wrong getting audio in MessageAudioCardState', - data: { - 'widget.messageEvent.messageDisplayLangCode': - messageEvent.messageDisplayLangCode, - }, - ); - if (_disposed) return; - audioState.value = AsyncState.error(e); - } - } - - Future fetchTranslation() async { - if (l1Code == null || - translationState.value is AsyncLoading || - translationState.value is AsyncLoaded) { - return; - } - - try { - translationState.value = const AsyncState.loading(); - final rep = await messageEvent.l1Respresentation(); - if (_disposed) return; - translationState.value = AsyncState.loaded(rep.text); - } catch (e, s) { - ErrorHandler.logError( - e: e, - s: s, - m: 'Error fetching translation', - data: { - 'l1Code': l1Code, - 'messageEvent': messageEvent.event.toJson(), - }, - ); - if (_disposed) return; - translationState.value = AsyncState.error(e); - } - } - - Future fetchTranscription() async { - try { - if (transcriptionState.value is AsyncLoading || - transcriptionState.value is AsyncLoaded) { - // If a transcription is already in progress or finished, don't fetch again - return; - } - - if (l1Code == null || l2Code == null) { - transcriptionState.value = const AsyncState.error( - 'Language code or message event is null', - ); - return; - } - - final resp = await messageEvent.getSpeechToText( - l1Code!, - l2Code!, - ); - - if (_disposed) return; - if (resp == null) { - transcriptionState.value = const AsyncState.error( - 'Transcription response is null', - ); - return; - } - transcriptionState.value = AsyncState.loaded(resp); - } catch (err, s) { - ErrorHandler.logError( - e: err, - s: s, - data: {}, - ); - if (_disposed) return; - transcriptionState.value = AsyncState.error(err); - } - } - - Future fetchSpeechTranslation() async { - if (l1Code == null || - l2Code == null || - speechTranslationState.value is AsyncLoading || - speechTranslationState.value is AsyncLoaded) { - return; - } - - if (transcriptionState.value is AsyncError) { - speechTranslationState.value = AsyncState.error( - (transcriptionState.value as AsyncError).error, - ); - return; - } - - try { - speechTranslationState.value = const AsyncState.loading(); - - if (transcriptionState.value is AsyncIdle || - transcriptionState.value is AsyncLoading) { - await fetchTranscription(); - if (_disposed) return; - if (transcriptionState.value is! AsyncLoaded) { - throw Exception('Transcription is null'); - } - } - - final translation = await messageEvent.sttTranslationByLanguageGlobal( - langCode: l1Code!, - l1Code: l1Code!, - l2Code: l2Code!, - ); - if (translation == null) { - throw Exception('Translation is null'); - } - - if (_disposed) return; - speechTranslationState.value = AsyncState.loaded(translation.translation); - } catch (err, s) { - ErrorHandler.logError( - e: err, - s: s, - data: {}, - ); - if (_disposed) return; - speechTranslationState.value = AsyncState.error(err); - } - } - - bool get isError { - switch (selectedMode.value) { - case SelectMode.audio: - return audioState.value is AsyncError; - case SelectMode.translate: - return translationState.value is AsyncError; - case SelectMode.speechTranslation: - return speechTranslationState.value is AsyncError; - default: - return false; - } - } - - bool get isLoading { - switch (selectedMode.value) { - case SelectMode.audio: - return audioState.value is AsyncLoading; - case SelectMode.translate: - return translationState.value is AsyncLoading; - case SelectMode.speechTranslation: - return speechTranslationState.value is AsyncLoading; - default: - return false; - } - } + Future fetchAudio() => _audioLoader.load(); + Future fetchTranslation() => _translationLoader.load(); + Future fetchTranscription() => _transcriptLoader.load(); + Future fetchSpeechTranslation() => _sttTranslationLoader.load(); }