ember/fluffychat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore: add SST translation event

Browse source
This commit is contained in:
ggurdin 2025-06-11 12:12:26 -04:00
parent 99fd9f9cb0
commit 296ddef06d
No known key found for this signature in database
GPG key ID: A01CB41737CBB478
9 changed files with 321 additions and 144 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
lib/pangea/events/constants/pangea_event_types.dart
View file

@ -16,6 +16,7 @@ class PangeaEventTypes {
static const tokens = "pangea.tokens";
static const choreoRecord = "pangea.record";
static const representation = "pangea.representation";
static const sttTranslation = "pangea.stt_translation";
// static const vocab = "p.vocab";
static const roomInfo = "pangea.roomtopic";

52
lib/pangea/events/controllers/message_data_controller.dart
View file

@ -13,6 +13,7 @@ import 'package:fluffychat/pangea/common/utils/error_handler.dart';
import 'package:fluffychat/pangea/events/constants/pangea_event_types.dart';
import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart';
import 'package:fluffychat/pangea/events/models/representation_content_model.dart';
import 'package:fluffychat/pangea/events/models/stt_translation_model.dart';
import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart';
import 'package:fluffychat/pangea/events/repo/token_api_models.dart';
import 'package:fluffychat/pangea/extensions/pangea_room_extension.dart';
@ -24,6 +25,7 @@ class MessageDataController extends BaseController {
final Map<int, Future<TokensResponseModel>> _tokensCache = {};
final Map<int, Future<PangeaRepresentation>> _representationCache = {};
final Map<int, Future<SttTranslationModel>> _sttTranslationCache = {};
late Timer _cacheTimer;
MessageDataController(PangeaController pangeaController) {
@ -42,6 +44,7 @@ class MessageDataController extends BaseController {
void _clearCache() {
_tokensCache.clear();
_representationCache.clear();
_sttTranslationCache.clear();
debugPrint("message data cache cleared.");
}
@ -219,4 +222,53 @@ class MessageDataController extends BaseController {
);
}
}
Future<SttTranslationModel> getSttTranslation({
required String? repEventId,
required FullTextTranslationRequestModel req,
required Room? room,
}) =>
_sttTranslationCache[req.hashCode] ??= _getSttTranslation(
repEventId: repEventId,
req: req,
room: room,
).catchError((e, s) {
_sttTranslationCache.remove(req.hashCode);
return Future<SttTranslationModel>.error(e, s);
});
Future<SttTranslationModel> _getSttTranslation({
required String? repEventId,
required FullTextTranslationRequestModel req,
required Room? room,
}) async {
final res = await FullTextTranslationRepo.translate(
accessToken: _pangeaController.userController.accessToken,
request: req,
);
final translation = SttTranslationModel(
translation: res.bestTranslation,
langCode: req.tgtLang,
);
if (repEventId != null && room != null) {
room
.sendPangeaEvent(
content: translation.toJson(),
parentEventId: repEventId,
type: PangeaEventTypes.sttTranslation,
)
.catchError(
(e) => ErrorHandler.logError(
m: "error in _getSttTranslation.sendPangeaEvent",
e: e,
s: StackTrace.current,
data: req.toJson(),
),
);
}
return translation;
}
}

47
lib/pangea/events/event_wrappers/pangea_message_event.dart
View file

@ -16,6 +16,7 @@ import 'package:fluffychat/pangea/events/event_wrappers/pangea_representation_ev
import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart';
import 'package:fluffychat/pangea/events/models/pangea_token_model.dart';
import 'package:fluffychat/pangea/events/models/representation_content_model.dart';
import 'package:fluffychat/pangea/events/models/stt_translation_model.dart';
import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart';
import 'package:fluffychat/pangea/morphs/morph_features_enum.dart';
import 'package:fluffychat/pangea/practice_activities/activity_type_enum.dart';
@ -281,32 +282,11 @@ class PangeaMessageEvent {
?.content
.speechToText;
if (speechToTextLocal != null) return speechToTextLocal;
if (speechToTextLocal != null) {
return speechToTextLocal;
}
final matrixFile = await _event.downloadAndDecryptAttachment();
// Pangea#
// File? file;
// TODO: Test on mobile and see if we need this case, doeesn't seem so
// if (!kIsWeb) {
// final tempDir = await getTemporaryDirectory();
// final fileName = Uri.encodeComponent(
// // #Pangea
// // widget.event.attachmentOrThumbnailMxcUrl()!.pathSegments.last,
// widget.messageEvent.event
// .attachmentOrThumbnailMxcUrl()!
// .pathSegments
// .last,
// // Pangea#
// );
// file = File('${tempDir.path}/${fileName}_${matrixFile.name}');
// await file.writeAsBytes(matrixFile.bytes);
// }
// audioFile = file;
debugPrint("mimeType ${matrixFile.mimeType}");
debugPrint("encoding ${mimeTypeToAudioEncoding(matrixFile.mimeType)}");
final SpeechToTextModel response =
await MatrixState.pangeaController.speechToText.get(
@ -341,6 +321,25 @@ class PangeaMessageEvent {
return response;
}
Future<SttTranslationModel?> sttTranslationByLanguageGlobal({
required String langCode,
required String l1Code,
required String l2Code,
}) async {
if (!representations.any(
(element) => element.content.speechToText != null,
)) {
await getSpeechToText(l1Code, l2Code);
}
final rep = representations.firstWhereOrNull(
(element) => element.content.speechToText != null,
);
if (rep == null) return null;
return rep.getSttTranslation(userL1: l1Code, userL2: l2Code);
}
PangeaMessageTokens? _tokensSafe(Map<String, dynamic>? content) {
try {
if (content == null) return null;

67
lib/pangea/events/event_wrappers/pangea_representation_event.dart
View file

@ -12,11 +12,13 @@ import 'package:sentry_flutter/sentry_flutter.dart';
import 'package:fluffychat/pangea/choreographer/event_wrappers/pangea_choreo_event.dart';
import 'package:fluffychat/pangea/choreographer/models/choreo_record.dart';
import 'package:fluffychat/pangea/choreographer/models/language_detection_model.dart';
import 'package:fluffychat/pangea/choreographer/repo/full_text_translation_repo.dart';
import 'package:fluffychat/pangea/common/utils/error_handler.dart';
import 'package:fluffychat/pangea/events/constants/pangea_event_types.dart';
import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart';
import 'package:fluffychat/pangea/events/models/pangea_token_model.dart';
import 'package:fluffychat/pangea/events/models/representation_content_model.dart';
import 'package:fluffychat/pangea/events/models/stt_translation_model.dart';
import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart';
import 'package:fluffychat/pangea/events/repo/token_api_models.dart';
import 'package:fluffychat/pangea/learning_settings/constants/language_constants.dart';
@ -210,6 +212,71 @@ class RepresentationEvent {
);
}
List<SttTranslationModel> get sttTranslations {
if (content.speechToText == null) return [];
if (_event == null) {
Sentry.addBreadcrumb(
Breadcrumb(
message: "_event and _sttTranslations both null",
),
);
return [];
}
final Set<Event> sttEvents = _event!.aggregatedEvents(
timeline,
PangeaEventTypes.sttTranslation,
);
if (sttEvents.isEmpty) return [];
final List<SttTranslationModel> sttTranslations = [];
for (final event in sttEvents) {
try {
sttTranslations.add(
SttTranslationModel.fromJson(event.content),
);
} catch (e) {
Sentry.addBreadcrumb(
Breadcrumb(
message: "Failed to parse STT translation",
data: {
"eventID": event.eventId,
"content": event.content,
"error": e.toString(),
},
),
);
}
}
return sttTranslations;
}
Future<SttTranslationModel> getSttTranslation({
required String userL1,
required String userL2,
}) async {
if (content.speechToText == null) {
throw Exception(
"RepresentationEvent.getSttTranslation called on a representation without speechToText",
);
}
final local = sttTranslations.firstWhereOrNull((t) => t.langCode == userL1);
if (local != null) return local;
return MatrixState.pangeaController.messageData.getSttTranslation(
repEventId: _event?.eventId,
room: _event?.room,
req: FullTextTranslationRequestModel(
text: content.speechToText!.transcript.text,
tgtLang: userL1,
userL2: userL2,
userL1: userL1,
),
);
}
ChoreoRecord? get choreo {
if (_choreo != null) return _choreo;

42
lib/pangea/events/event_wrappers/pangea_tokens_event.dart
View file

@ -1,42 +0,0 @@
import 'dart:developer';
import 'package:flutter/foundation.dart';
import 'package:matrix/matrix.dart';
import 'package:fluffychat/pangea/common/utils/error_handler.dart';
import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart';
import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart';
import '../constants/pangea_event_types.dart';
class TokensEvent {
Event event;
PangeaMessageTokens? _content;
TokensEvent({required this.event}) {
if (event.type != PangeaEventTypes.tokens) {
throw Exception(
"${event.type} should not be used to make a TokensEvent",
);
}
}
PangeaMessageTokens? get _pangeaMessageTokens {
try {
_content ??= event.getPangeaContent<PangeaMessageTokens>();
return _content!;
} catch (err, s) {
debugger(when: kDebugMode);
ErrorHandler.logError(
e: err,
s: s,
data: {
"event": event.toJson(),
},
);
return null;
}
}
PangeaMessageTokens? get tokens => _pangeaMessageTokens;
}

23
lib/pangea/events/models/stt_translation_model.dart Normal file
View file

@ -0,0 +1,23 @@
class SttTranslationModel {
final String translation;
final String langCode;
SttTranslationModel({
required this.translation,
required this.langCode,
});
factory SttTranslationModel.fromJson(Map<String, dynamic> json) {
return SttTranslationModel(
translation: json['translation'] as String,
langCode: json['lang_code'] as String,
);
}
Map<String, dynamic> toJson() {
return {
'translation': translation,
'lang_code': langCode,
};
}
}

59
lib/pangea/toolbar/widgets/message_selection_overlay.dart
View file

@ -30,6 +30,7 @@ import 'package:fluffychat/pangea/toolbar/controllers/text_to_speech_controller.
import 'package:fluffychat/pangea/toolbar/controllers/tts_controller.dart';
import 'package:fluffychat/pangea/toolbar/enums/message_mode_enum.dart';
import 'package:fluffychat/pangea/toolbar/enums/reading_assistance_mode_enum.dart';
import 'package:fluffychat/pangea/toolbar/models/speech_to_text_models.dart';
import 'package:fluffychat/pangea/toolbar/reading_assistance_input_row/morph_selection.dart';
import 'package:fluffychat/pangea/toolbar/widgets/message_selection_positioner.dart';
import 'package:fluffychat/pangea/toolbar/widgets/reading_assistance_content.dart';
@ -91,12 +92,15 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
ReadingAssistanceMode? readingAssistanceMode; // default mode
bool showTranslation = false;
String? translationText;
String? transcriptionText;
SpeechToTextModel? transcription;
String? transcriptionError;
bool showTranslation = false;
String? translation;
bool showSpeechTranslation = false;
String? speechTranslation;
double maxWidth = AppConfig.toolbarMinWidth;
/////////////////////////////////////
@ -574,33 +578,50 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
);
}
void setShowTranslation(bool show, String? translation) {
if (showTranslation == show) return;
if (show && translation == null) return;
void setTranslation(String value) {
if (mounted) {
setState(() {
showTranslation = show;
translationText = show ? translation : null;
});
setState(() => translation = value);
}
}
void setTranscriptionText(String transcription) {
void setShowTranslation(bool show) {
if (!mounted) return;
if (translation == null) {
setState(() => showTranslation = false);
}
if (showTranslation == show) return;
setState(() => showTranslation = show);
}
void setSpeechTranslation(String value) {
if (mounted) {
setState(() => speechTranslation = value);
}
}
void setShowSpeechTranslation(bool show) {
if (!mounted) return;
if (speechTranslation == null) {
setState(() => showSpeechTranslation = false);
}
if (showSpeechTranslation == show) return;
setState(() => showSpeechTranslation = show);
}
void setTranscription(SpeechToTextModel value) {
if (mounted) {
setState(() {
transcriptionError = null;
transcriptionText = transcription;
transcription = value;
});
}
}
void setTranscriptionError(String error) {
void setTranscriptionError(String value) {
if (mounted) {
setState(() {
transcriptionText = null;
transcriptionError = error;
});
setState(() => transcriptionError = value);
}
}

16
lib/pangea/toolbar/widgets/overlay_message.dart
View file

@ -135,10 +135,13 @@ class OverlayMessage extends StatelessWidget {
event.numberEmotes <= 3);
final showTranslation = overlayController.showTranslation &&
overlayController.translationText != null;
overlayController.translation != null;
final showTranscription = pangeaMessageEvent?.isAudioMessage == true;
final showSpeechTranslation = overlayController.showSpeechTranslation &&
overlayController.speechTranslation != null;
final content = Container(
decoration: BoxDecoration(
borderRadius: BorderRadius.circular(
@ -296,10 +299,11 @@ class OverlayMessage extends StatelessWidget {
),
],
)
: overlayController.transcriptionText != null
: overlayController.transcription != null
? SingleChildScrollView(
child: Text(
overlayController.transcriptionText!,
overlayController
.transcription!.transcript.text,
style: AppConfig.messageTextStyle(
event,
textColor,
@ -323,7 +327,7 @@ class OverlayMessage extends StatelessWidget {
},
)
: content,
if (showTranslation)
if (showTranslation || showSpeechTranslation)
Container(
width: messageWidth,
constraints: const BoxConstraints(
@ -338,7 +342,9 @@ class OverlayMessage extends StatelessWidget {
),
child: SingleChildScrollView(
child: Text(
overlayController.translationText!,
showTranslation
? overlayController.translation!
: overlayController.speechTranslation!,
style: AppConfig.messageTextStyle(
event,
textColor,

158
lib/pangea/toolbar/widgets/select_mode_buttons.dart
View file

@ -18,7 +18,6 @@ import 'package:fluffychat/pangea/common/widgets/pressable_button.dart';
import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart';
import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart';
import 'package:fluffychat/pangea/events/models/representation_content_model.dart';
import 'package:fluffychat/pangea/toolbar/models/speech_to_text_models.dart';
import 'package:fluffychat/pangea/toolbar/widgets/message_audio_card.dart';
import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart';
import 'package:fluffychat/widgets/matrix.dart';
@ -88,10 +87,12 @@ class SelectModeButtonsState extends State<SelectModeButtons> {
StreamSubscription? _onAudioPositionChanged;
bool _isLoadingTranslation = false;
PangeaRepresentation? _repEvent;
String? _translationError;
SpeechToTextModel? _speechToTextResponse;
bool _isLoadingSpeechTranslation = false;
String? _speechTranslationError;
Completer<String>? _transcriptionCompleter;
@override
void initState() {
@ -113,7 +114,7 @@ class SelectModeButtonsState extends State<SelectModeButtons> {
});
if (messageEvent?.isAudioMessage == true) {
_loadTranscription();
_fetchTranscription();
}
}
@ -129,9 +130,9 @@ class SelectModeButtonsState extends State<SelectModeButtons> {
widget.overlayController.pangeaMessageEvent;
String? get l1Code =>
MatrixState.pangeaController.languageController.activeL1Code();
MatrixState.pangeaController.languageController.userL1?.langCodeShort;
String? get l2Code =>
MatrixState.pangeaController.languageController.activeL2Code();
MatrixState.pangeaController.languageController.userL2?.langCodeShort;
void _clear() {
setState(() {
@ -140,10 +141,8 @@ class SelectModeButtonsState extends State<SelectModeButtons> {
});
widget.overlayController.updateSelectedSpan(null);
if (_selectedMode == SelectMode.translate) {
widget.overlayController.setShowTranslation(false, null);
}
widget.overlayController.setShowTranslation(false);
widget.overlayController.setShowSpeechTranslation(false);
}
Future<void> _updateMode(SelectMode? mode) async {
@ -177,7 +176,13 @@ class SelectModeButtonsState extends State<SelectModeButtons> {
}
if (_selectedMode == SelectMode.translate) {
await _loadTranslation();
await _fetchTranslation();
widget.overlayController.setShowTranslation(true);
}
if (_selectedMode == SelectMode.speechTranslation) {
await _fetchSpeechTranslation();
widget.overlayController.setShowSpeechTranslation(true);
}
}
@ -265,67 +270,68 @@ class SelectModeButtonsState extends State<SelectModeButtons> {
}
}
Future<void> _fetchRepresentation() async {
if (l1Code == null || messageEvent == null || _repEvent != null) {
Future<void> _fetchTranslation() async {
if (l1Code == null ||
messageEvent == null ||
widget.overlayController.translation != null) {
return;
}
_repEvent = messageEvent!.representationByLanguage(l1Code!)?.content;
if (_repEvent == null && mounted) {
_repEvent = await messageEvent?.representationByLanguageGlobal(
try {
if (mounted) setState(() => _isLoadingTranslation = true);
PangeaRepresentation? rep =
messageEvent!.representationByLanguage(l1Code!)?.content;
rep ??= await messageEvent?.representationByLanguageGlobal(
langCode: l1Code!,
);
widget.overlayController.setTranslation(rep!.text);
} catch (e, s) {
_translationError = e.toString();
ErrorHandler.logError(
e: e,
s: s,
m: 'Error fetching translation',
data: {
'l1Code': l1Code,
'messageEvent': messageEvent?.event.toJson(),
},
);
} finally {
if (mounted) setState(() => _isLoadingTranslation = false);
}
}
Future<void> _fetchTranscription() async {
if (l1Code == null || messageEvent == null || _repEvent != null) {
return;
}
_speechToTextResponse ??= await messageEvent!.getSpeechToText(
l1Code!,
l2Code!,
);
}
Future<void> _loadTranslation() async {
if (!mounted) return;
setState(() => _isLoadingTranslation = true);
try {
await _fetchRepresentation();
if (_repEvent == null) {
throw "No representation found for the selected language.";
if (_transcriptionCompleter != null) {
// If a transcription is already in progress, wait for it to complete
await _transcriptionCompleter!.future;
return;
}
widget.overlayController.setShowTranslation(
true,
_repEvent!.text,
);
} catch (err) {
_translationError = err.toString();
ErrorHandler.logError(
e: err,
data: {},
);
}
_transcriptionCompleter = Completer<String>();
if (l1Code == null || messageEvent == null) {
_transcriptionCompleter?.completeError(
'Language code or message event is null',
);
return;
}
if (mounted) {
setState(() => _isLoadingTranslation = false);
}
}
Future<void> _loadTranscription() async {
try {
await _fetchTranscription();
widget.overlayController.setTranscriptionText(
_speechToTextResponse!.transcript.text,
final resp = await messageEvent!.getSpeechToText(
l1Code!,
l2Code!,
);
widget.overlayController.setTranscription(resp!);
_transcriptionCompleter?.complete(resp.transcript.text);
} catch (err) {
widget.overlayController.setTranscriptionError(
err.toString(),
);
_transcriptionCompleter?.completeError(err);
ErrorHandler.logError(
e: err,
data: {},
@ -333,12 +339,54 @@ class SelectModeButtonsState extends State<SelectModeButtons> {
}
}
Future<void> _fetchSpeechTranslation() async {
if (messageEvent == null ||
l1Code == null ||
l2Code == null ||
widget.overlayController.speechTranslation != null) {
return;
}
try {
setState(() => _isLoadingSpeechTranslation = true);
if (widget.overlayController.transcription == null) {
await _fetchTranscription();
if (widget.overlayController.transcription == null) {
throw Exception('Transcription is null');
}
}
final translation = await messageEvent!.sttTranslationByLanguageGlobal(
langCode: l1Code!,
l1Code: l1Code!,
l2Code: l2Code!,
);
if (translation == null) {
throw Exception('Translation is null');
}
widget.overlayController.setSpeechTranslation(translation.translation);
} catch (err, s) {
debugPrint("Error fetching speech translation: $err, $s");
_speechTranslationError = err.toString();
ErrorHandler.logError(
e: err,
data: {},
);
} finally {
if (mounted) setState(() => _isLoadingSpeechTranslation = false);
}
}
bool get _isError {
switch (_selectedMode) {
case SelectMode.audio:
return _audioError != null;
case SelectMode.translate:
return _translationError != null;
case SelectMode.speechTranslation:
return _speechTranslationError != null;
default:
return false;
}
@ -350,6 +398,8 @@ class SelectModeButtonsState extends State<SelectModeButtons> {
return _isLoadingAudio;
case SelectMode.translate:
return _isLoadingTranslation;
case SelectMode.speechTranslation:
return _isLoadingSpeechTranslation;
default:
return false;
}