From 9cb155fcf1aa3ddabc5caac4c4e63efc92589799 Mon Sep 17 00:00:00 2001 From: ggurdin Date: Wed, 3 Dec 2025 15:22:45 -0500 Subject: [PATCH] reorganize pangea rep / pangea message event files --- lib/pages/chat/chat.dart | 2 +- .../activity_summary_analytics_model.dart | 2 +- .../event_wrappers/pangea_message_event.dart | 811 ++++++++---------- .../pangea_representation_event.dart | 392 ++++----- .../events/repo/language_detection_repo.dart | 91 +- .../repo/language_detection_request.dart | 12 + .../toolbar/widgets/message_audio_card.dart | 15 +- .../widgets/message_selection_overlay.dart | 10 +- .../widgets/select_mode_controller.dart | 26 +- 9 files changed, 609 insertions(+), 752 deletions(-) diff --git a/lib/pages/chat/chat.dart b/lib/pages/chat/chat.dart index 4bc911fa8..199392716 100644 --- a/lib/pages/chat/chat.dart +++ b/lib/pages/chat/chat.dart @@ -2149,7 +2149,7 @@ class ChatController extends State ownMessage: true, ); - final stt = await messageEvent.getSpeechToText( + final stt = await messageEvent.requestSpeechToText( MatrixState.pangeaController.languageController.userL1?.langCodeShort ?? LanguageKeys.unknownLanguage, MatrixState.pangeaController.languageController.userL2?.langCodeShort ?? diff --git a/lib/pangea/activity_summary/activity_summary_analytics_model.dart b/lib/pangea/activity_summary/activity_summary_analytics_model.dart index dc6b80581..32344a7c4 100644 --- a/lib/pangea/activity_summary/activity_summary_analytics_model.dart +++ b/lib/pangea/activity_summary/activity_summary_analytics_model.dart @@ -45,7 +45,7 @@ class ActivitySummaryAnalyticsModel { } void addMessageConstructs(PangeaMessageEvent event) { - final uses = event.originalSent?.vocabAndMorphUses(); + final uses = event.originalSent?.vocabAndMorphUses; if (uses == null || uses.isEmpty) return; addConstructs(event.senderId, uses); } diff --git a/lib/pangea/events/event_wrappers/pangea_message_event.dart b/lib/pangea/events/event_wrappers/pangea_message_event.dart index cbdc481f2..230974851 100644 --- a/lib/pangea/events/event_wrappers/pangea_message_event.dart +++ b/lib/pangea/events/event_wrappers/pangea_message_event.dart @@ -4,13 +4,14 @@ import 'dart:ui'; import 'package:flutter/foundation.dart'; +import 'package:async/async.dart'; import 'package:collection/collection.dart'; -import 'package:matrix/matrix.dart'; -import 'package:sentry_flutter/sentry_flutter.dart'; +import 'package:matrix/matrix.dart' hide Result; import 'package:fluffychat/pangea/choreographer/choreo_record_model.dart'; import 'package:fluffychat/pangea/common/constants/model_keys.dart'; import 'package:fluffychat/pangea/events/event_wrappers/pangea_representation_event.dart'; +import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart'; import 'package:fluffychat/pangea/events/models/representation_content_model.dart'; import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart'; import 'package:fluffychat/pangea/events/repo/language_detection_repo.dart'; @@ -72,12 +73,11 @@ class PangeaMessageEvent { bool get isAudioMessage => _event.messageType == MessageTypes.Audio; - String? get mimetype { - if (!isAudioMessage) return null; - final Map? info = _event.content.tryGetMap("info"); - if (info == null) return null; - return info["mime_type"] ?? info["mimetype"]; - } + String? get _l2Code => + MatrixState.pangeaController.languageController.activeL2Code(); + + String? get _l1Code => + MatrixState.pangeaController.languageController.userL1?.langCode; Event? _latestEditCache; Event get _latestEdit => _latestEditCache ??= _event @@ -92,131 +92,6 @@ class PangeaMessageEvent { .firstOrNull ?? _event; - void updateLatestEdit() { - _latestEditCache = null; - _representations = null; - } - - Future getMatrixAudioFile( - String langCode, - ) async { - final RepresentationEvent? rep = representationByLanguage(langCode); - final tokensResp = await rep?.tokensGlobal( - senderId, - originServerTs, - ); - - final TextToSpeechRequest params = TextToSpeechRequest( - text: rep?.content.text ?? body, - tokens: tokensResp?.result?.map((t) => t.text).toList() ?? [], - langCode: langCode, - userL1: l1Code ?? LanguageKeys.unknownLanguage, - userL2: l2Code ?? LanguageKeys.unknownLanguage, - ); - - final TextToSpeechResponse response = - await MatrixState.pangeaController.textToSpeech.get( - params, - ); - - final audioBytes = base64.decode(response.audioContent); - final eventIdParam = _event.eventId; - final fileName = - "audio_for_${eventIdParam}_$langCode.${response.fileExtension}"; - - final file = PangeaAudioFile( - bytes: audioBytes, - name: fileName, - mimeType: response.mimeType, - duration: response.durationMillis, - waveform: response.waveform, - tokens: response.ttsTokens, - ); - - sendAudioEvent(file, response, rep?.text ?? body, langCode); - - return file; - } - - Future sendAudioEvent( - PangeaAudioFile file, - TextToSpeechResponse response, - String text, - String langCode, - ) async { - final String? eventId = await room.sendFileEvent( - file, - inReplyTo: _event, - extraContent: { - 'info': { - ...file.info, - 'duration': response.durationMillis, - }, - 'org.matrix.msc3245.voice': {}, - 'org.matrix.msc1767.audio': { - 'duration': response.durationMillis, - 'waveform': response.waveform, - }, - ModelKey.transcription: - response.toPangeaAudioEventData(text, langCode).toJson(), - }, - ); - - debugPrint("eventId in getTextToSpeechGlobal $eventId"); - final Event? audioEvent = - eventId != null ? await room.getEventById(eventId) : null; - - if (audioEvent == null) { - return null; - } - allAudio.add(audioEvent); - return audioEvent; - } - - Event? getTextToSpeechLocal(String langCode, String text) { - return allAudio.firstWhereOrNull( - (event) { - try { - // Safely access - final dataMap = event.content.tryGetMap(ModelKey.transcription); - - if (dataMap == null) { - return false; - } - - // old text to speech content will not have TTSToken data - // we want to disregard them and just generate new ones - // for that, we'll return false if 'tokens' are null - // while in-development, we'll pause here to inspect - // debugger can be removed after we're sure it's working - if (dataMap['tokens'] == null) { - // events before today will definitely not have the tokens - debugger( - when: kDebugMode && - event.originServerTs.isAfter(DateTime(2024, 10, 16)), - ); - return false; - } - - final PangeaAudioEventData audioData = - PangeaAudioEventData.fromJson(dataMap as dynamic); - - // Check if both language code and text match - return audioData.langCode == langCode && audioData.text == text; - } catch (e, s) { - debugger(when: kDebugMode); - ErrorHandler.logError( - e: e, - s: s, - data: {}, - m: "error parsing data in getTextToSpeechLocal", - ); - return false; - } - }, - ); - } - // get audio events that are related to this event Set get allAudio => _latestEdit .aggregatedEvents( @@ -230,159 +105,25 @@ class PangeaMessageEvent { null; }).toSet(); - SpeechToTextResponseModel? getSpeechToTextLocal() { - final rawBotTranscription = - event.content.tryGetMap(ModelKey.botTranscription); - - if (rawBotTranscription != null) { - try { - return SpeechToTextResponseModel.fromJson( - Map.from(rawBotTranscription), - ); - } catch (err, s) { - ErrorHandler.logError( - e: err, - s: s, - data: { - "event": _event.toJson(), - }, - m: "error parsing botTranscription", - ); - return null; - } - } - - return representations - .firstWhereOrNull( - (element) => element.content.speechToText != null, - ) - ?.content - .speechToText; - } - - Future getSpeechToText( - String l1Code, - String l2Code, - ) async { - if (!isAudioMessage) { - throw 'Calling getSpeechToText on non-audio message'; - } - - final rawBotTranscription = - event.content.tryGetMap(ModelKey.botTranscription); - if (rawBotTranscription != null) { - final SpeechToTextResponseModel botTranscription = - SpeechToTextResponseModel.fromJson( - Map.from(rawBotTranscription), - ); - - _representations ??= []; - _representations!.add( - RepresentationEvent( - timeline: timeline, - parentMessageEvent: _event, - content: PangeaRepresentation( - langCode: botTranscription.langCode, - text: botTranscription.transcript.text, - originalSent: false, - originalWritten: false, - speechToText: botTranscription, - ), - ), - ); - - return botTranscription; - } - - final SpeechToTextResponseModel? speechToTextLocal = representations - .firstWhereOrNull( - (element) => element.content.speechToText != null, - ) - ?.content - .speechToText; - - if (speechToTextLocal != null) { - return speechToTextLocal; - } - - final matrixFile = await _event.downloadAndDecryptAttachment(); - - final result = await SpeechToTextRepo.get( - MatrixState.pangeaController.userController.accessToken, - SpeechToTextRequestModel( - audioContent: matrixFile.bytes, - audioEvent: _event, - config: SpeechToTextAudioConfigModel( - encoding: mimeTypeToAudioEncoding(matrixFile.mimeType), - sampleRateHertz: 22050, - userL1: l1Code, - userL2: l2Code, - ), - ), - ); - - if (result.error != null) { - throw Exception( - "Error getting speech to text: ${result.error}", - ); - } - - final SpeechToTextResponseModel response = result.result!; - _representations?.add( - RepresentationEvent( - timeline: timeline, - parentMessageEvent: _event, - content: PangeaRepresentation( - langCode: response.langCode, - text: response.transcript.text, - originalSent: false, - originalWritten: false, - speechToText: response, - ), - ), - ); - - return response; - } - - Future sttTranslationByLanguageGlobal({ - required String langCode, - required String l1Code, - required String l2Code, - }) async { - if (!representations.any( - (element) => element.content.speechToText != null, - )) { - await getSpeechToText(l1Code, l2Code); - } - - final rep = representations.firstWhereOrNull( - (element) => element.content.speechToText != null, - ); - - if (rep == null) { - throw Exception("No speech to text representation found"); - } - - final resp = await rep.getSttTranslation(userL1: l1Code, userL2: l2Code); - return resp.translation; - } - - PangeaMessageTokens? _tokensSafe(Map? content) { - try { - if (content == null) return null; - return PangeaMessageTokens.fromJson(content); - } catch (e, s) { - debugger(when: kDebugMode); - ErrorHandler.logError( - e: e, - s: s, - data: content ?? {}, - m: "error parsing tokensSent", - ); - return null; - } - } + List get _repEvents => _latestEdit + .aggregatedEvents( + timeline, + PangeaEventTypes.representation, + ) + .map( + (e) => RepresentationEvent( + event: e, + parentMessageEvent: _event, + timeline: timeline, + ), + ) + .sorted( + (a, b) { + if (a.event == null) return -1; + if (b.event == null) return 1; + return b.event!.originServerTs.compareTo(a.event!.originServerTs); + }, + ).toList(); ChoreoRecordModel? get _embeddedChoreo { try { @@ -402,6 +143,22 @@ class PangeaMessageEvent { } } + PangeaMessageTokens? _tokensSafe(Map? content) { + try { + if (content == null) return null; + return PangeaMessageTokens.fromJson(content); + } catch (e, s) { + debugger(when: kDebugMode); + ErrorHandler.logError( + e: e, + s: s, + data: content ?? {}, + m: "error parsing tokensSent", + ); + return null; + } + } + List? _representations; List get representations { if (_representations != null) return _representations!; @@ -419,18 +176,6 @@ class PangeaMessageEvent { choreo: _embeddedChoreo, timeline: timeline, ); - if (_latestEdit.content[ModelKey.choreoRecord] == null) { - Sentry.addBreadcrumb( - Breadcrumb( - message: "originalSent created without _event or _choreo", - data: { - "eventId": _latestEdit.eventId, - "room": _latestEdit.room.id, - "sender": _latestEdit.senderId, - }, - ), - ); - } // If originalSent has no tokens, there is not way to generate a tokens event // and send it as a related event, since original sent has not eventID to set @@ -480,155 +225,10 @@ class PangeaMessageEvent { } } - _representations!.addAll( - _latestEdit - .aggregatedEvents( - timeline, - PangeaEventTypes.representation, - ) - .map( - (e) => RepresentationEvent( - event: e, - parentMessageEvent: _event, - timeline: timeline, - ), - ) - .sorted( - (a, b) { - //TODO - test with edited events to make sure this is working - if (a.event == null) return -1; - if (b.event == null) return 1; - return b.event!.originServerTs.compareTo(a.event!.originServerTs); - }, - ).toList(), - ); - + _representations!.addAll(_repEvents); return _representations!; } - RepresentationEvent? representationByLanguage( - String langCode, { - bool Function(RepresentationEvent)? filter, - }) => - representations.firstWhereOrNull( - (element) => - element.langCode.split("-")[0] == langCode.split("-")[0] && - (filter?.call(element) ?? true), - ); - - Future representationByDetectedLanguage() async { - LanguageDetectionResponse? resp; - try { - resp = await LanguageDetectionRepo.get( - MatrixState.pangeaController.userController.accessToken, - request: LanguageDetectionRequest( - text: _latestEdit.body, - senderl1: l1Code, - senderl2: l2Code, - ), - ); - } catch (e, s) { - ErrorHandler.logError( - e: e, - s: s, - data: { - "event": _event.toJson(), - }, - ); - return null; - } - - final langCode = resp.detections.firstOrNull?.langCode; - if (langCode == null) return null; - if (langCode == originalSent?.langCode) { - return originalSent?.event?.eventId; - } - - // clear representations cache so the new representation event can be added when next requested - _representations = null; - - final res = await FullTextTranslationRepo.get( - MatrixState.pangeaController.userController.accessToken, - FullTextTranslationRequestModel( - text: originalSent?.content.text ?? _latestEdit.body, - srcLang: originalSent?.langCode, - tgtLang: langCode, - userL2: l2Code ?? LanguageKeys.unknownLanguage, - userL1: l1Code ?? LanguageKeys.unknownLanguage, - ), - ); - - if (res.isError) return null; - final repEvent = await room.sendPangeaEvent( - content: PangeaRepresentation( - langCode: langCode, - text: res.result!, - originalSent: originalSent == null, - originalWritten: false, - ).toJson(), - parentEventId: eventId, - type: PangeaEventTypes.representation, - ); - return repEvent?.eventId; - } - - Future l1Respresentation() async { - if (l1Code == null || l2Code == null) { - throw Exception("Missing language codes"); - } - - final includedIT = - (originalSent?.choreo?.endedWithIT(originalSent!.text) ?? false) && - !(originalSent?.choreo?.includedIGC ?? true); - - RepresentationEvent? rep; - if (!includedIT) { - // if the message didn't go through translation, get any l1 rep - rep = representationByLanguage(l1Code!); - } else { - // if the message went through translation, get the non-original - // l1 rep since originalWritten could contain some l2 words - // (https://github.com/pangeachat/client/issues/3591) - rep = representationByLanguage( - l1Code!, - filter: (rep) => !rep.content.originalWritten, - ); - } - - if (rep != null) return rep.content.text; - - final String srcLang = includedIT - ? (originalWritten?.langCode ?? l1Code!) - : (originalSent?.langCode ?? l2Code!); - - // clear representations cache so the new representation event can be added when next requested - _representations = null; - - final resp = await FullTextTranslationRepo.get( - MatrixState.pangeaController.userController.accessToken, - FullTextTranslationRequestModel( - text: includedIT ? originalWrittenContent : messageDisplayText, - srcLang: srcLang, - tgtLang: l1Code!, - userL2: l2Code!, - userL1: l1Code!, - ), - ); - - if (resp.isError) throw resp.error!; - room.sendPangeaEvent( - content: PangeaRepresentation( - langCode: l1Code!, - text: resp.result!, - originalSent: false, - originalWritten: false, - ).toJson(), - parentEventId: eventId, - type: PangeaEventTypes.representation, - ); - return resp.result!; - } - RepresentationEvent? get originalSent => representations .firstWhereOrNull((element) => element.content.originalSent); @@ -646,12 +246,6 @@ class PangeaMessageEvent { return written ?? body; } - String? get l2Code => - MatrixState.pangeaController.languageController.activeL2Code(); - - String? get l1Code => - MatrixState.pangeaController.languageController.userL1?.langCode; - String get messageDisplayLangCode { if (isAudioMessage) { final stt = getSpeechToTextLocal(); @@ -665,7 +259,7 @@ class PangeaMessageEvent { final String? originalLangCode = originalSent?.langCode; - final String? langCode = immersionMode ? l2Code : originalLangCode; + final String? langCode = immersionMode ? _l2Code : originalLangCode; return langCode ?? LanguageKeys.unknownLanguage; } @@ -681,4 +275,319 @@ class PangeaMessageEvent { PLanguageStore.rtlLanguageCodes.contains(messageDisplayLangCode) ? TextDirection.rtl : TextDirection.ltr; + + void updateLatestEdit() { + _latestEditCache = null; + _representations = null; + } + + RepresentationEvent? representationByLanguage( + String langCode, { + bool Function(RepresentationEvent)? filter, + }) => + representations.firstWhereOrNull( + (element) => + element.langCode.split("-")[0] == langCode.split("-")[0] && + (filter?.call(element) ?? true), + ); + + Event? getTextToSpeechLocal(String langCode, String text) { + for (final audio in allAudio) { + final dataMap = audio.content.tryGetMap(ModelKey.transcription); + if (dataMap == null || !dataMap.containsKey('tokens')) continue; + + try { + final PangeaAudioEventData audioData = PangeaAudioEventData.fromJson( + dataMap as dynamic, + ); + + if (audioData.langCode == langCode && audioData.text == text) { + return audio; + } + } catch (e, s) { + debugger(when: kDebugMode); + ErrorHandler.logError( + e: e, + s: s, + data: { + "event": audio.toJson(), + }, + m: "error parsing data in getTextToSpeechLocal", + ); + } + } + return null; + } + + SpeechToTextResponseModel? getSpeechToTextLocal() { + final rawBotTranscription = + event.content.tryGetMap(ModelKey.botTranscription); + + if (rawBotTranscription != null) { + try { + return SpeechToTextResponseModel.fromJson( + Map.from(rawBotTranscription), + ); + } catch (err, s) { + ErrorHandler.logError( + e: err, + s: s, + data: { + "event": _event.toJson(), + }, + m: "error parsing botTranscription", + ); + return null; + } + } + + return representations + .firstWhereOrNull( + (element) => element.content.speechToText != null, + ) + ?.content + .speechToText; + } + + Future requestTextToSpeech( + String langCode, + ) async { + final local = getTextToSpeechLocal(langCode, messageDisplayText); + if (local != null) { + final file = await local.getPangeaAudioFile(); + if (file != null) return file; + } + + final rep = representationByLanguage(langCode); + final tokensResp = await rep?.requestTokens(); + final request = TextToSpeechRequest( + text: rep?.content.text ?? body, + tokens: tokensResp?.result?.map((t) => t.text).toList() ?? [], + langCode: langCode, + userL1: _l1Code ?? LanguageKeys.unknownLanguage, + userL2: _l2Code ?? LanguageKeys.unknownLanguage, + ); + + final response = await MatrixState.pangeaController.textToSpeech.get( + request, + ); + + final audioBytes = base64.decode(response.audioContent); + final fileName = + "audio_for_${_event.eventId}_$langCode.${response.fileExtension}"; + + final file = PangeaAudioFile( + bytes: audioBytes, + name: fileName, + mimeType: response.mimeType, + duration: response.durationMillis, + waveform: response.waveform, + tokens: response.ttsTokens, + ); + + room.sendFileEvent( + file, + inReplyTo: _event, + extraContent: { + 'info': { + ...file.info, + 'duration': response.durationMillis, + }, + 'org.matrix.msc3245.voice': {}, + 'org.matrix.msc1767.audio': { + 'duration': response.durationMillis, + 'waveform': response.waveform, + }, + ModelKey.transcription: response + .toPangeaAudioEventData(rep?.text ?? body, langCode) + .toJson(), + }, + ).then((eventId) async { + final Event? audioEvent = + eventId != null ? await room.getEventById(eventId) : null; + + if (audioEvent != null) { + allAudio.add(audioEvent); + } + }); + + return file; + } + + Future requestSpeechToText( + String l1Code, + String l2Code, + ) async { + if (!isAudioMessage) { + throw 'Calling getSpeechToText on non-audio message'; + } + + final speechToTextLocal = getSpeechToTextLocal(); + if (speechToTextLocal != null) { + return speechToTextLocal; + } + + final matrixFile = await _event.downloadAndDecryptAttachment(); + final result = await SpeechToTextRepo.get( + MatrixState.pangeaController.userController.accessToken, + SpeechToTextRequestModel( + audioContent: matrixFile.bytes, + audioEvent: _event, + config: SpeechToTextAudioConfigModel( + encoding: mimeTypeToAudioEncoding(matrixFile.mimeType), + sampleRateHertz: 22050, + userL1: l1Code, + userL2: l2Code, + ), + ), + ); + + if (result.error != null) { + throw Exception( + "Error getting speech to text: ${result.error}", + ); + } + + _representations = null; + return result.result!; + } + + Future requestSttTranslation({ + required String langCode, + required String l1Code, + required String l2Code, + }) async { + if (!representations.any( + (element) => element.content.speechToText != null, + )) { + await requestSpeechToText(l1Code, l2Code); + } + + final rep = representations.firstWhereOrNull( + (element) => element.content.speechToText != null, + ); + + if (rep == null) { + throw Exception("No speech to text representation found"); + } + + final resp = await rep.requestSttTranslation( + userL1: l1Code, + userL2: l2Code, + ); + return resp.translation; + } + + Future requestRepresentationByDetectedLanguage() async { + LanguageDetectionResponse? resp; + final result = await LanguageDetectionRepo.get( + MatrixState.pangeaController.userController.accessToken, + LanguageDetectionRequest( + text: _latestEdit.body, + senderl1: _l1Code, + senderl2: _l2Code, + ), + ); + + if (result.isError) return null; + resp = result.result!; + + final langCode = resp.detections.firstOrNull?.langCode; + if (langCode == null) return null; + if (langCode == originalSent?.langCode) { + return originalSent?.event?.eventId; + } + + final res = await _requestRepresentation( + originalSent?.content.text ?? _latestEdit.body, + langCode, + originalSent?.langCode ?? LanguageKeys.unknownLanguage, + originalSent: originalSent == null, + ); + + if (res.isError) return null; + return _sendRepresentationEvent(res.result!); + } + + Future requestRespresentationByL1() async { + if (_l1Code == null || _l2Code == null) { + throw Exception("Missing language codes"); + } + + final includedIT = + (originalSent?.choreo?.endedWithIT(originalSent!.text) ?? false) && + !(originalSent?.choreo?.includedIGC ?? true); + + RepresentationEvent? rep; + if (!includedIT) { + // if the message didn't go through translation, get any l1 rep + rep = representationByLanguage(_l1Code!); + } else { + // if the message went through translation, get the non-original + // l1 rep since originalWritten could contain some l2 words + // (https://github.com/pangeachat/client/issues/3591) + rep = representationByLanguage( + _l1Code!, + filter: (rep) => !rep.content.originalWritten, + ); + } + + if (rep != null) return rep.content.text; + + final String srcLang = includedIT + ? (originalWritten?.langCode ?? _l1Code!) + : (originalSent?.langCode ?? _l2Code!); + + final resp = await _requestRepresentation( + includedIT ? originalWrittenContent : messageDisplayText, + _l1Code!, + srcLang, + ); + + if (resp.isError) throw resp.error!; + _sendRepresentationEvent(resp.result!); + return resp.result!.text; + } + + Future> _requestRepresentation( + String text, + String targetLang, + String sourceLang, { + bool originalSent = false, + }) async { + _representations = null; + + final res = await FullTextTranslationRepo.get( + MatrixState.pangeaController.userController.accessToken, + FullTextTranslationRequestModel( + text: text, + srcLang: sourceLang, + tgtLang: targetLang, + userL2: _l2Code ?? LanguageKeys.unknownLanguage, + userL1: _l1Code ?? LanguageKeys.unknownLanguage, + ), + ); + + return res.isError + ? Result.error(res.error!) + : Result.value( + PangeaRepresentation( + langCode: targetLang, + text: res.result!, + originalSent: originalSent, + originalWritten: false, + ), + ); + } + + Future _sendRepresentationEvent( + PangeaRepresentation representation, + ) async { + final repEvent = await room.sendPangeaEvent( + content: representation.toJson(), + parentEventId: eventId, + type: PangeaEventTypes.representation, + ); + return repEvent?.eventId; + } } diff --git a/lib/pangea/events/event_wrappers/pangea_representation_event.dart b/lib/pangea/events/event_wrappers/pangea_representation_event.dart index 06b28c16f..14c4ab0e9 100644 --- a/lib/pangea/events/event_wrappers/pangea_representation_event.dart +++ b/lib/pangea/events/event_wrappers/pangea_representation_event.dart @@ -7,7 +7,6 @@ import 'package:flutter/foundation.dart'; import 'package:async/async.dart'; import 'package:collection/collection.dart'; import 'package:matrix/matrix.dart' hide Result; -import 'package:matrix/src/utils/markdown.dart'; import 'package:sentry_flutter/sentry_flutter.dart'; import 'package:fluffychat/pangea/analytics_misc/constructs_model.dart'; @@ -25,9 +24,6 @@ import 'package:fluffychat/pangea/events/repo/token_api_models.dart'; import 'package:fluffychat/pangea/events/repo/tokens_repo.dart'; import 'package:fluffychat/pangea/extensions/pangea_room_extension.dart'; import 'package:fluffychat/pangea/learning_settings/constants/language_constants.dart'; -import 'package:fluffychat/pangea/morphs/morph_features_enum.dart'; -import 'package:fluffychat/pangea/morphs/parts_of_speech_enum.dart'; -import 'package:fluffychat/pangea/practice_activities/activity_type_enum.dart'; import 'package:fluffychat/pangea/translation/full_text_translation_repo.dart'; import 'package:fluffychat/pangea/translation/full_text_translation_request_model.dart'; import 'package:fluffychat/widgets/future_loading_dialog.dart'; @@ -62,6 +58,33 @@ class RepresentationEvent { Event? get event => _event; + String get text => content.text; + + String get langCode => content.langCode; + + List? get detections => _tokens?.detections; + + Set get tokenEvents => + _event?.aggregatedEvents( + timeline, + PangeaEventTypes.tokens, + ) ?? + {}; + + Set get sttEvents => + _event?.aggregatedEvents( + timeline, + PangeaEventTypes.sttTranslation, + ) ?? + {}; + + Set get choreoEvents => + _event?.aggregatedEvents( + timeline, + PangeaEventTypes.choreoRecord, + ) ?? + {}; + // Note: in the case where the event is the originalSent or originalWritten event, // the content will be set on initialization by the PangeaMessageEvent // Otherwise, the content will be fetched from the event where it is stored in content[type] @@ -71,50 +94,134 @@ class RepresentationEvent { return _content!; } - String get text => content.text; - - String get langCode => content.langCode; - - bool get botAuthored => - content.originalSent == false && content.originalWritten == false; - - List? get detections => _tokens?.detections; - List? get tokens { if (_tokens != null) return _tokens!.tokens; if (_event == null) return null; - final Set tokenEvents = _event?.aggregatedEvents( - timeline, - PangeaEventTypes.tokens, - ) ?? - {}; - if (tokenEvents.isEmpty) return null; _tokens = tokenEvents.last.getPangeaContent(); return _tokens?.tokens; } - Future>> tokensGlobal( - String senderID, - DateTime timestamp, - ) async { - if (tokens != null) return Result.value(tokens!); + ChoreoRecordModel? get choreo { + if (_choreo != null) return _choreo; - if (_event == null && timestamp.isAfter(DateTime(2024, 9, 25))) { + if (_event == null) { Sentry.addBreadcrumb( Breadcrumb( - message: - 'representation with no _event and no tokens got tokens directly. This means an original_sent with no tokens. This should not happen in messages sent after September 25', - data: { - 'content': content.toJson(), - 'event': _event?.toJson(), - 'timestamp': timestamp.toIso8601String(), - 'senderID': senderID, - }, + message: "_event and _choreo both null", ), ); + return null; } + + if (choreoEvents.isEmpty) return null; + if (choreoEvents.length > 1) { + debugger(when: kDebugMode); + ErrorHandler.logError( + m: 'should not have more than one choreoEvent per representation ${_event?.eventId}', + s: StackTrace.current, + data: {"event": _event?.toJson()}, + ); + } + + return ChoreoEvent(event: choreoEvents.first).content; + } + + List get sttTranslations { + if (content.speechToText == null) return []; + if (_event == null) { + Sentry.addBreadcrumb( + Breadcrumb( + message: "_event and _sttTranslations both null", + ), + ); + return []; + } + + if (sttEvents.isEmpty) return []; + final List sttTranslations = []; + for (final event in sttEvents) { + try { + sttTranslations.add( + SttTranslationModel.fromJson(event.content), + ); + } catch (e) { + Sentry.addBreadcrumb( + Breadcrumb( + message: "Failed to parse STT translation", + data: { + "eventID": event.eventId, + "content": event.content, + "error": e.toString(), + }, + ), + ); + } + } + return sttTranslations; + } + + List get vocabAndMorphUses { + if (tokens == null || tokens!.isEmpty) { + return []; + } + + final metadata = ConstructUseMetaData( + roomId: parentMessageEvent.room.id, + timeStamp: parentMessageEvent.originServerTs, + eventId: parentMessageEvent.eventId, + ); + + return content.vocabAndMorphUses( + tokens: tokens!, + metadata: metadata, + choreo: choreo, + ); + } + + /// Finds the closest non-punctuation token to the given token. + PangeaToken? getClosestNonPunctToken(PangeaToken token) { + // If it's not punctuation, it's already the closest. + if (token.pos != "PUNCT") return token; + + final list = tokens; + if (list == null) return null; + + final index = list.indexOf(token); + if (index == -1) return null; + + PangeaToken? left; + PangeaToken? right; + + // Scan left + for (int i = index - 1; i >= 0; i--) { + if (list[i].pos != "PUNCT") { + left = list[i]; + break; + } + } + + // Scan right + for (int i = index + 1; i < list.length; i++) { + if (list[i].pos != "PUNCT") { + right = list[i]; + break; + } + } + + if (left == null) return right; + if (right == null) return left; + + // Choose the nearest by distance + final leftDistance = token.start - left.end; + final rightDistance = right.start - token.end; + + return leftDistance < rightDistance ? left : right; + } + + Future>> requestTokens() async { + if (tokens != null) return Result.value(tokens!); final res = await TokensRepo.get( MatrixState.pangeaController.userController.accessToken, TokensRequestModel( @@ -140,90 +247,12 @@ class RepresentationEvent { ); } - if (res.isError) { - return Result.error(res.error!); - } else { - return Result.value(res.result!.tokens); - } + return res.isError + ? Result.error(res.error!) + : Result.value(res.result!.tokens); } - Future sendTokensEvent( - String repEventID, - Room room, - String userl1, - String userl2, - ) async { - if (tokens != null) return; - if (_event == null) { - ErrorHandler.logError( - e: "Called getTokensEvent with no _event", - data: {}, - ); - return; - } - - final resp = await TokensRepo.get( - MatrixState.pangeaController.userController.accessToken, - TokensRequestModel( - fullText: text, - langCode: langCode, - senderL1: userl1, - senderL2: userl2, - ), - ); - - if (resp.isError) return; - room.sendPangeaEvent( - content: PangeaMessageTokens( - tokens: resp.result!.tokens, - detections: resp.result!.detections, - ).toJson(), - parentEventId: _event!.eventId, - type: PangeaEventTypes.tokens, - ); - } - - List get sttTranslations { - if (content.speechToText == null) return []; - if (_event == null) { - Sentry.addBreadcrumb( - Breadcrumb( - message: "_event and _sttTranslations both null", - ), - ); - return []; - } - - final Set sttEvents = _event!.aggregatedEvents( - timeline, - PangeaEventTypes.sttTranslation, - ); - - if (sttEvents.isEmpty) return []; - final List sttTranslations = []; - for (final event in sttEvents) { - try { - sttTranslations.add( - SttTranslationModel.fromJson(event.content), - ); - } catch (e) { - Sentry.addBreadcrumb( - Breadcrumb( - message: "Failed to parse STT translation", - data: { - "eventID": event.eventId, - "content": event.content, - "error": e.toString(), - }, - ), - ); - } - } - - return sttTranslations; - } - - Future getSttTranslation({ + Future requestSttTranslation({ required String userL1, required String userL2, }) async { @@ -263,137 +292,4 @@ class RepresentationEvent { return translation; } - - ChoreoRecordModel? get choreo { - if (_choreo != null) return _choreo; - - if (_event == null) { - Sentry.addBreadcrumb( - Breadcrumb( - message: "_event and _choreo both null", - ), - ); - return null; - } - - final Set choreoMatrixEvents = - _event?.aggregatedEvents(timeline, PangeaEventTypes.choreoRecord) ?? {}; - - if (choreoMatrixEvents.isEmpty) return null; - - if (choreoMatrixEvents.length > 1) { - debugger(when: kDebugMode); - ErrorHandler.logError( - m: 'should not have more than one choreoEvent per representation ${_event?.eventId}', - s: StackTrace.current, - data: {"event": _event?.toJson()}, - ); - } - - _choreo = ChoreoEvent(event: choreoMatrixEvents.first).content; - - return _choreo; - } - - String? formatBody() { - return markdown(content.text); - } - - /// Finds the closest non-punctuation token to the given token. - /// - /// This method checks if the provided token is a punctuation token. If it is not, - /// it returns the token itself. If the token is a punctuation token, it searches - /// through the list of tokens to find the closest non-punctuation token either to - /// the left or right of the given token. - /// - /// If both left and right non-punctuation tokens are found, it returns the one - /// that is closest to the given token. If only one of them is found, it returns - /// that token. If no non-punctuation tokens are found, it returns null. - /// - /// - Parameters: - /// - token: The token for which to find the closest non-punctuation token. - /// - /// - Returns: The closest non-punctuation token, or null if no such token exists. - PangeaToken? getClosestNonPunctToken(PangeaToken token) { - if (token.pos != "PUNCT") return token; - if (tokens == null) return null; - final index = tokens!.indexOf(token); - if (index > -1) { - final leftTokens = tokens!.sublist(0, index); - final rightTokens = tokens!.sublist(index + 1); - final leftMostToken = leftTokens.lastWhereOrNull( - (element) => element.pos != "PUNCT", - ); - final rightMostToken = rightTokens.firstWhereOrNull( - (element) => element.pos != "PUNCT", - ); - - if (leftMostToken != null && rightMostToken != null) { - final leftDistance = token.start - leftMostToken.end; - final rightDistance = rightMostToken.start - token.end; - return leftDistance < rightDistance ? leftMostToken : rightMostToken; - } else if (leftMostToken != null) { - return leftMostToken; - } else if (rightMostToken != null) { - return rightMostToken; - } - } - return null; - } - - List get tokensToSave => - tokens?.where((token) => token.lemma.saveVocab).toList() ?? []; - - // List get allTokenMorphsToConstructIdentifiers => tokens?.map((t) => t.morphConstructIds).toList() ?? - // []; - - /// get allTokenMorphsToConstructIdentifiers - Set get morphFeatureSetToPractice => - MorphFeaturesEnum.values.where((feature) { - // pos is always included - if (feature == MorphFeaturesEnum.Pos) { - return true; - } - return tokens?.any((token) => token.morph.containsKey(feature)) ?? - false; - }).toSet(); - - Set posSetToPractice(ActivityTypeEnum a) => - PartOfSpeechEnum.values.where((pos) { - // some pos are not eligible for practice at all - if (!pos.eligibleForPractice(a)) { - return false; - } - return tokens?.any( - (token) => token.pos.toLowerCase() == pos.name.toLowerCase(), - ) ?? - false; - }).toSet(); - - List tagsByFeature(MorphFeaturesEnum feature) { - return tokens - ?.where((t) => t.morph.containsKey(feature)) - .map((t) => t.morph[feature]) - .cast() - .toList() ?? - []; - } - - List vocabAndMorphUses() { - if (tokens == null || tokens!.isEmpty) { - return []; - } - - final metadata = ConstructUseMetaData( - roomId: parentMessageEvent.room.id, - timeStamp: parentMessageEvent.originServerTs, - eventId: parentMessageEvent.eventId, - ); - - return content.vocabAndMorphUses( - tokens: tokens!, - metadata: metadata, - choreo: choreo, - ); - } } diff --git a/lib/pangea/events/repo/language_detection_repo.dart b/lib/pangea/events/repo/language_detection_repo.dart index 97ee1a5e7..c4f6a6864 100644 --- a/lib/pangea/events/repo/language_detection_repo.dart +++ b/lib/pangea/events/repo/language_detection_repo.dart @@ -1,18 +1,47 @@ import 'dart:convert'; +import 'package:async/async.dart'; import 'package:http/http.dart'; import 'package:fluffychat/pangea/common/config/environment.dart'; import 'package:fluffychat/pangea/common/network/requests.dart'; import 'package:fluffychat/pangea/common/network/urls.dart'; +import 'package:fluffychat/pangea/common/utils/error_handler.dart'; import 'package:fluffychat/pangea/events/repo/language_detection_request.dart'; import 'package:fluffychat/pangea/events/repo/language_detection_response.dart'; +class _LanguageDetectionCacheItem { + final Future data; + final DateTime timestamp; + + const _LanguageDetectionCacheItem({ + required this.data, + required this.timestamp, + }); +} + class LanguageDetectionRepo { - static Future get( - String? accessToken, { - required LanguageDetectionRequest request, - }) async { + static final Map _cache = {}; + static const Duration _cacheDuration = Duration(minutes: 10); + + static Future> get( + String accessToken, + LanguageDetectionRequest request, + ) { + final cached = _getCached(request); + if (cached != null) { + return _getResult(request, cached); + } + + final future = _fetch(accessToken, request); + _setCached(request, future); + return _getResult(request, future); + } + + static Future _fetch( + String accessToken, + LanguageDetectionRequest request, + ) async { final Requests req = Requests( accessToken: accessToken, choreoApiKey: Environment.choreoApiKey, @@ -22,12 +51,54 @@ class LanguageDetectionRepo { body: request.toJson(), ); - final Map json = - jsonDecode(utf8.decode(res.bodyBytes).toString()); + if (res.statusCode != 200) { + throw Exception( + 'Failed to detect language: ${res.statusCode} ${res.reasonPhrase}', + ); + } - final LanguageDetectionResponse response = - LanguageDetectionResponse.fromJson(json); - - return response; + return LanguageDetectionResponse.fromJson( + jsonDecode(utf8.decode(res.bodyBytes)), + ); } + + static Future> _getResult( + LanguageDetectionRequest request, + Future future, + ) async { + try { + final res = await future; + return Result.value(res); + } catch (e, s) { + _cache.remove(request.hashCode.toString()); + ErrorHandler.logError( + e: e, + s: s, + data: request.toJson(), + ); + return Result.error(e); + } + } + + static Future? _getCached( + LanguageDetectionRequest request, + ) { + final cacheKeys = [..._cache.keys]; + for (final key in cacheKeys) { + if (DateTime.now().difference(_cache[key]!.timestamp) >= _cacheDuration) { + _cache.remove(key); + } + } + + return _cache[request.hashCode.toString()]?.data; + } + + static void _setCached( + LanguageDetectionRequest request, + Future response, + ) => + _cache[request.hashCode.toString()] = _LanguageDetectionCacheItem( + data: response, + timestamp: DateTime.now(), + ); } diff --git a/lib/pangea/events/repo/language_detection_request.dart b/lib/pangea/events/repo/language_detection_request.dart index eb28d6507..803d5db83 100644 --- a/lib/pangea/events/repo/language_detection_request.dart +++ b/lib/pangea/events/repo/language_detection_request.dart @@ -16,4 +16,16 @@ class LanguageDetectionRequest { 'sender_l2': senderl2, }; } + + @override + int get hashCode => text.hashCode ^ senderl1.hashCode ^ senderl2.hashCode; + + @override + bool operator ==(Object other) { + if (identical(this, other)) return true; + return other is LanguageDetectionRequest && + other.text == text && + other.senderl1 == senderl1 && + other.senderl2 == senderl2; + } } diff --git a/lib/pangea/toolbar/widgets/message_audio_card.dart b/lib/pangea/toolbar/widgets/message_audio_card.dart index dcd48c4bf..626c89e80 100644 --- a/lib/pangea/toolbar/widgets/message_audio_card.dart +++ b/lib/pangea/toolbar/widgets/message_audio_card.dart @@ -10,7 +10,6 @@ import 'package:fluffychat/pages/chat/events/audio_player.dart'; import 'package:fluffychat/pangea/analytics_misc/text_loading_shimmer.dart'; import 'package:fluffychat/pangea/common/utils/error_handler.dart'; import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart'; -import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart'; import 'package:fluffychat/pangea/toolbar/controllers/text_to_speech_controller.dart'; class MessageAudioCard extends StatefulWidget { @@ -42,19 +41,9 @@ class MessageAudioCardState extends State { setState(() => _isLoading = true); try { - final String langCode = widget.messageEvent.messageDisplayLangCode; - final Event? localEvent = widget.messageEvent.getTextToSpeechLocal( - langCode, - widget.messageEvent.messageDisplayText, + audioFile = await widget.messageEvent.requestTextToSpeech( + widget.messageEvent.messageDisplayLangCode, ); - - if (localEvent != null) { - audioFile = await localEvent.getPangeaAudioFile(); - } else { - audioFile = await widget.messageEvent.getMatrixAudioFile( - langCode, - ); - } debugPrint("audio file is now: $audioFile. setting starts and ends..."); if (mounted) setState(() => _isLoading = false); } catch (e, s) { diff --git a/lib/pangea/toolbar/widgets/message_selection_overlay.dart b/lib/pangea/toolbar/widgets/message_selection_overlay.dart index 9753d9718..c631e858c 100644 --- a/lib/pangea/toolbar/widgets/message_selection_overlay.dart +++ b/lib/pangea/toolbar/widgets/message_selection_overlay.dart @@ -112,12 +112,7 @@ class MessageOverlayController extends State } if (repEvent?.event != null) { - await repEvent!.sendTokensEvent( - repEvent.event!.eventId, - widget._event.room, - MatrixState.pangeaController.languageController.userL1!.langCode, - MatrixState.pangeaController.languageController.userL2!.langCode, - ); + await repEvent!.requestTokens(); } } catch (e, s) { debugger(when: kDebugMode); @@ -268,7 +263,8 @@ class MessageOverlayController extends State pangeaMessageEvent.messageDisplayRepresentation; if (repEvent != null) return repEvent; - final eventID = await pangeaMessageEvent.representationByDetectedLanguage(); + final eventID = + await pangeaMessageEvent.requestRepresentationByDetectedLanguage(); if (eventID == null) return null; final event = await widget._event.room.getEventById(eventID); diff --git a/lib/pangea/toolbar/widgets/select_mode_controller.dart b/lib/pangea/toolbar/widgets/select_mode_controller.dart index fd7a760ae..cfe98d4c5 100644 --- a/lib/pangea/toolbar/widgets/select_mode_controller.dart +++ b/lib/pangea/toolbar/widgets/select_mode_controller.dart @@ -8,7 +8,6 @@ import 'package:path_provider/path_provider.dart'; import 'package:fluffychat/pangea/common/utils/async_state.dart'; import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart'; -import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart'; import 'package:fluffychat/pangea/speech_to_text/speech_to_text_response_model.dart'; import 'package:fluffychat/pangea/toolbar/widgets/message_audio_card.dart'; import 'package:fluffychat/pangea/toolbar/widgets/select_mode_buttons.dart'; @@ -19,7 +18,7 @@ class _TranscriptionLoader extends AsyncLoader { _TranscriptionLoader(this.messageEvent) : super(); @override - Future fetch() => messageEvent.getSpeechToText( + Future fetch() => messageEvent.requestSpeechToText( MatrixState.pangeaController.languageController.userL1!.langCodeShort, MatrixState.pangeaController.languageController.userL2!.langCodeShort, ); @@ -30,7 +29,7 @@ class _STTTranslationLoader extends AsyncLoader { _STTTranslationLoader(this.messageEvent) : super(); @override - Future fetch() => messageEvent.sttTranslationByLanguageGlobal( + Future fetch() => messageEvent.requestSttTranslation( langCode: MatrixState .pangeaController.languageController.userL1!.langCodeShort, l1Code: MatrixState @@ -45,7 +44,7 @@ class _TranslationLoader extends AsyncLoader { _TranslationLoader(this.messageEvent) : super(); @override - Future fetch() => messageEvent.l1Respresentation(); + Future fetch() => messageEvent.requestRespresentationByL1(); } class _AudioLoader extends AsyncLoader<(PangeaAudioFile, File?)> { @@ -54,25 +53,10 @@ class _AudioLoader extends AsyncLoader<(PangeaAudioFile, File?)> { @override Future<(PangeaAudioFile, File?)> fetch() async { - final String langCode = messageEvent.messageDisplayLangCode; - - final Event? localEvent = messageEvent.getTextToSpeechLocal( - langCode, - messageEvent.messageDisplayText, + final audioBytes = await messageEvent.requestTextToSpeech( + messageEvent.messageDisplayLangCode, ); - PangeaAudioFile? audioBytes; - if (localEvent != null) { - audioBytes = await localEvent.getPangeaAudioFile(); - } else { - audioBytes = await messageEvent.getMatrixAudioFile( - langCode, - ); - } - if (audioBytes == null) { - throw Exception('Audio bytes are null'); - } - File? audioFile; if (!kIsWeb) { final tempDir = await getTemporaryDirectory();