From 1f5c722bcd92afe9c511dfed85ac2a5450cfd36f Mon Sep 17 00:00:00 2001 From: ggurdin Date: Wed, 11 Jun 2025 13:09:04 -0400 Subject: [PATCH] add audio analytics --- lib/pages/chat/chat.dart | 207 ++++++++++++------ .../toolbar/models/speech_to_text_models.dart | 25 +++ .../toolbar/widgets/select_mode_buttons.dart | 1 + 3 files changed, 169 insertions(+), 64 deletions(-) diff --git a/lib/pages/chat/chat.dart b/lib/pages/chat/chat.dart index 19a6181ab..bcf34389f 100644 --- a/lib/pages/chat/chat.dart +++ b/lib/pages/chat/chat.dart @@ -51,6 +51,7 @@ import 'package:fluffychat/pangea/events/models/representation_content_model.dar import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart'; import 'package:fluffychat/pangea/extensions/pangea_room_extension.dart'; import 'package:fluffychat/pangea/instructions/instructions_enum.dart'; +import 'package:fluffychat/pangea/learning_settings/constants/language_constants.dart'; import 'package:fluffychat/pangea/learning_settings/widgets/p_language_dialog.dart'; import 'package:fluffychat/pangea/toolbar/enums/message_mode_enum.dart'; import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart'; @@ -924,31 +925,7 @@ class ChatController extends State ), ]; - final newGrammarConstructs = - pangeaController.getAnalytics.newConstructCount( - constructs, - ConstructTypeEnum.morph, - ); - - final newVocabConstructs = - pangeaController.getAnalytics.newConstructCount( - constructs, - ConstructTypeEnum.vocab, - ); - - OverlayUtil.showOverlay( - overlayKey: "msg_analytics_feedback_$msgEventId", - followerAnchor: Alignment.bottomRight, - targetAnchor: Alignment.topRight, - context: context, - child: MessageAnalyticsFeedback( - overlayId: "msg_analytics_feedback_$msgEventId", - newGrammarConstructs: newGrammarConstructs, - newVocabConstructs: newVocabConstructs, - ), - transformTargetId: msgEventId, - ignorePointer: true, - ); + _showAnalyticsFeedback(constructs, msgEventId); pangeaController.putAnalytics.setState( AnalyticsStream( @@ -1130,44 +1107,54 @@ class ChatController extends State name: result.fileName ?? audioFile.path, ); - await room.sendFileEvent( - file, - inReplyTo: replyEvent, - extraContent: { - 'info': { - ...file.info, - 'duration': result.duration, - }, - 'org.matrix.msc3245.voice': {}, - 'org.matrix.msc1767.audio': { - 'duration': result.duration, - 'waveform': result.waveform, - }, - }, - // #Pangea - // ).catchError((e) { - ).catchError((e, s) { - ErrorHandler.logError( - e: e, - s: s, - data: { - 'roomId': roomId, - 'file': file.name, - 'duration': result.duration, - 'waveform': result.waveform, - }, - ); - // Pangea# - scaffoldMessenger.showSnackBar( - SnackBar( - content: Text( - (e as Object).toLocalizedString(context), - ), - ), - ); - return null; - }); - // #Pangea + await room + .sendFileEvent( + file, + inReplyTo: replyEvent, + extraContent: { + 'info': { + ...file.info, + 'duration': result.duration, + }, + 'org.matrix.msc3245.voice': {}, + 'org.matrix.msc1767.audio': { + 'duration': result.duration, + 'waveform': result.waveform, + }, + }, + // #Pangea + ) + .then(_sendVoiceMessageAnalytics) + .catchError((e, s) { + ErrorHandler.logError( + e: e, + s: s, + data: { + 'roomId': roomId, + 'file': file.name, + 'duration': result.duration, + 'waveform': result.waveform, + }, + ); + scaffoldMessenger.showSnackBar( + SnackBar( + content: Text( + (e as Object).toLocalizedString(context), + ), + ), + ); + return null; + }); + // ).catchError((e) { + // scaffoldMessenger.showSnackBar( + // SnackBar( + // content: Text( + // (e as Object).toLocalizedString(context), + // ), + // ), + // ); + // return null; + // }); // setState(() { // replyEvent = null; // }); @@ -1618,7 +1605,8 @@ class ChatController extends State if (timeline == null || events.any( (e) => e.aggregatedEvents(timeline!, RelationshipTypes.reaction).any( - (re) => re.content.tryGetMap('m.relates_to')?['key'] == emoji), + (re) => re.content.tryGetMap('m.relates_to')?['key'] == emoji, + ), )) { return; } @@ -2058,6 +2046,97 @@ class ChatController extends State return false; } } + + Future _sendVoiceMessageAnalytics(String? eventId) async { + if (eventId == null) { + ErrorHandler.logError( + e: Exception('eventID null in voiceMessageAction'), + s: StackTrace.current, + data: { + 'roomId': roomId, + }, + ); + return; + } + + final event = await room.getEventById(eventId); + if (event == null) { + ErrorHandler.logError( + e: Exception('Event not found after sending voice message'), + s: StackTrace.current, + data: { + 'roomId': roomId, + }, + ); + return; + } + + try { + final messageEvent = PangeaMessageEvent( + event: event, + timeline: timeline!, + ownMessage: true, + ); + + final stt = await messageEvent.getSpeechToText( + choreographer.l1Lang?.langCodeShort ?? LanguageKeys.unknownLanguage, + choreographer.l2Lang?.langCodeShort ?? LanguageKeys.unknownLanguage, + ); + if (stt == null || stt.transcript.sttTokens.isEmpty) return; + final constructs = stt.constructs(roomId, eventId); + if (constructs.isEmpty) return; + + _showAnalyticsFeedback(constructs, eventId); + MatrixState.pangeaController.putAnalytics.setState( + AnalyticsStream( + eventId: eventId, + targetID: eventId, + roomId: room.id, + constructs: constructs, + ), + ); + } catch (e, s) { + ErrorHandler.logError( + e: e, + s: s, + data: { + 'roomId': roomId, + 'eventId': eventId, + }, + ); + } + } + + void _showAnalyticsFeedback( + List constructs, + String eventId, + ) { + final newGrammarConstructs = + pangeaController.getAnalytics.newConstructCount( + constructs, + ConstructTypeEnum.morph, + ); + + final newVocabConstructs = pangeaController.getAnalytics.newConstructCount( + constructs, + ConstructTypeEnum.vocab, + ); + + OverlayUtil.showOverlay( + overlayKey: "msg_analytics_feedback_$eventId", + followerAnchor: Alignment.bottomRight, + targetAnchor: Alignment.topRight, + context: context, + child: MessageAnalyticsFeedback( + overlayId: "msg_analytics_feedback_$eventId", + newGrammarConstructs: newGrammarConstructs, + newVocabConstructs: newVocabConstructs, + ), + transformTargetId: eventId, + ignorePointer: true, + closePrevOverlay: false, + ); + } // Pangea# late final ValueNotifier _displayChatDetailsColumn; diff --git a/lib/pangea/toolbar/models/speech_to_text_models.dart b/lib/pangea/toolbar/models/speech_to_text_models.dart index d43f45643..1b7b676af 100644 --- a/lib/pangea/toolbar/models/speech_to_text_models.dart +++ b/lib/pangea/toolbar/models/speech_to_text_models.dart @@ -6,6 +6,8 @@ import 'package:flutter/material.dart'; import 'package:matrix/matrix.dart'; import 'package:fluffychat/config/app_config.dart'; +import 'package:fluffychat/pangea/analytics_misc/construct_use_type_enum.dart'; +import 'package:fluffychat/pangea/analytics_misc/constructs_model.dart'; import 'package:fluffychat/pangea/events/models/pangea_token_model.dart'; import 'package:fluffychat/pangea/toolbar/enums/audio_encoding_enum.dart'; @@ -230,4 +232,27 @@ class SpeechToTextModel { Map toJson() => { "results": results.map((e) => e.toJson()).toList(), }; + + List constructs( + String roomId, + String eventId, + ) { + final List constructs = []; + final metadata = ConstructUseMetaData( + roomId: roomId, + eventId: eventId, + timeStamp: DateTime.now(), + ); + for (final sstToken in transcript.sttTokens) { + final token = sstToken.token; + constructs.addAll( + token.allUses( + ConstructUseTypeEnum.pvm, + metadata, + ConstructUseTypeEnum.pvm.pointValue, + ), + ); + } + return constructs; + } } diff --git a/lib/pangea/toolbar/widgets/select_mode_buttons.dart b/lib/pangea/toolbar/widgets/select_mode_buttons.dart index 76cca02f3..2903eaa2e 100644 --- a/lib/pangea/toolbar/widgets/select_mode_buttons.dart +++ b/lib/pangea/toolbar/widgets/select_mode_buttons.dart @@ -138,6 +138,7 @@ class SelectModeButtonsState extends State { setState(() { _audioError = null; _translationError = null; + _speechTranslationError = null; }); widget.overlayController.updateSelectedSpan(null);