ember/fluffychat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #4772 from pangeachat/4749-translation-tool-on-all-messages

Browse source 
4749 translation tool on all messages
This commit is contained in:
ggurdin 2025-12-02 15:48:43 -05:00 • committed by GitHub
commit c47d8898de
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 222 additions and 343 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
lib/pages/chat/chat.dart
View file

@ -2130,7 +2130,7 @@ class ChatController extends State<ChatPageWithRoom>
MatrixState.pangeaController.languageController.userL2?.langCodeShort ??
LanguageKeys.unknownLanguage,
);
if (stt == null || stt.transcript.sttTokens.isEmpty) return;
if (stt.transcript.sttTokens.isEmpty) return;
final constructs = stt.constructs(roomId, eventId);
if (constructs.isEmpty) return;

48
lib/pangea/common/utils/async_state.dart
View file

@ -1,3 +1,7 @@
import 'package:flutter/material.dart';
import 'package:fluffychat/pangea/common/utils/error_handler.dart';
/// A generic sealed class that represents the state of an asynchronous operation.
sealed class AsyncState<T> {
/// Base constructor for all asynchronous state variants.
@ -53,3 +57,47 @@ class AsyncError<T> extends AsyncState<T> {
/// Creates an error [AsyncState] with an [error].
const AsyncError(this.error);
}
abstract class AsyncLoader<T> {
final ValueNotifier<AsyncState<T>> state = ValueNotifier(AsyncState.idle());
bool _disposed = false;
bool get isIdle => state.value is AsyncIdle<T>;
bool get isLoading => state.value is AsyncLoading<T>;
bool get isLoaded => state.value is AsyncLoaded<T>;
bool get isError => state.value is AsyncError<T>;
T? get value => isLoaded ? (state.value as AsyncLoaded<T>).value : null;
void dispose() {
_disposed = true;
state.dispose();
}
Future<T> fetch();
Future<void> load() async {
if (state.value is AsyncLoading || state.value is AsyncLoaded) {
// If already loading or loaded, do nothing.
return;
}
state.value = AsyncState.loading();
try {
final result = await fetch();
if (_disposed) return;
state.value = AsyncState.loaded(result);
} catch (e, s) {
ErrorHandler.logError(
e: e,
s: s,
data: {},
);
if (!_disposed) {
state.value = AsyncState.error(e);
}
}
}
}

52
lib/pangea/events/event_wrappers/pangea_message_event.dart
View file

@ -13,7 +13,6 @@ import 'package:fluffychat/pangea/common/constants/model_keys.dart';
import 'package:fluffychat/pangea/events/controllers/message_data_controller.dart';
import 'package:fluffychat/pangea/events/event_wrappers/pangea_representation_event.dart';
import 'package:fluffychat/pangea/events/models/representation_content_model.dart';
import 'package:fluffychat/pangea/events/models/stt_translation_model.dart';
import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart';
import 'package:fluffychat/pangea/events/repo/language_detection_repo.dart';
import 'package:fluffychat/pangea/events/repo/language_detection_request.dart';
@ -258,44 +257,20 @@ class PangeaMessageEvent {
.speechToText;
}
Future<SpeechToTextModel?> getSpeechToText(
Future<SpeechToTextModel> getSpeechToText(
String l1Code,
String l2Code,
) async {
if (!isAudioMessage) {
ErrorHandler.logError(
e: 'Calling getSpeechToText on non-audio message',
s: StackTrace.current,
data: {
"content": _event.content,
"eventId": _event.eventId,
"roomId": _event.roomId,
"userId": _event.room.client.userID,
"account_data": _event.room.client.accountData,
},
);
return null;
throw 'Calling getSpeechToText on non-audio message';
}
final rawBotTranscription =
event.content.tryGetMap(ModelKey.botTranscription);
if (rawBotTranscription != null) {
SpeechToTextModel botTranscription;
try {
botTranscription = SpeechToTextModel.fromJson(
Map<String, dynamic>.from(rawBotTranscription),
);
} catch (err, s) {
ErrorHandler.logError(
e: err,
s: s,
data: {
"event": _event.toJson(),
},
m: "error parsing botTranscription",
);
return null;
}
final SpeechToTextModel botTranscription = SpeechToTextModel.fromJson(
Map<String, dynamic>.from(rawBotTranscription),
);
_representations ??= [];
_representations!.add(
@ -361,7 +336,7 @@ class PangeaMessageEvent {
return response;
}
Future<SttTranslationModel?> sttTranslationByLanguageGlobal({
Future<String> sttTranslationByLanguageGlobal({
required String langCode,
required String l1Code,
required String l2Code,
@ -376,8 +351,12 @@ class PangeaMessageEvent {
(element) => element.content.speechToText != null,
);
if (rep == null) return null;
return rep.getSttTranslation(userL1: l1Code, userL2: l2Code);
if (rep == null) {
throw Exception("No speech to text representation found");
}
final resp = await rep.getSttTranslation(userL1: l1Code, userL2: l2Code);
return resp.translation;
}
PangeaMessageTokens? _tokensSafe(Map<String, dynamic>? content) {
@ -572,7 +551,7 @@ class PangeaMessageEvent {
);
}
Future<PangeaRepresentation> l1Respresentation() async {
Future<String> l1Respresentation() async {
if (l1Code == null || l2Code == null) {
throw Exception("Missing language codes");
}
@ -595,7 +574,7 @@ class PangeaMessageEvent {
);
}
if (rep != null) return rep.content;
if (rep != null) return rep.content.text;
final String srcLang = includedIT
? (originalWritten?.langCode ?? l1Code!)
@ -603,7 +582,7 @@ class PangeaMessageEvent {
// clear representations cache so the new representation event can be added when next requested
_representations = null;
return MessageDataController.getPangeaRepresentation(
final resp = await MessageDataController.getPangeaRepresentation(
req: FullTextTranslationRequestModel(
text: includedIT ? originalWrittenContent : messageDisplayText,
srcLang: srcLang,
@ -613,6 +592,7 @@ class PangeaMessageEvent {
),
messageEvent: _event,
);
return resp.text;
}
RepresentationEvent? get originalSent => representations

44
lib/pangea/toolbar/widgets/message_selection_overlay.dart
View file

@ -134,16 +134,9 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
}
}
/// Decides whether an _initialSelectedToken should be used
/// for a first practice activity on the word meaning
Future<void> _initializeSelectedToken() async {
// if there is no initial selected token, then we don't need to do anything
if (widget._initialSelectedToken == null) {
return;
}
updateSelectedSpan(widget._initialSelectedToken!.text);
}
void _initializeSelectedToken() => widget._initialSelectedToken != null
? updateSelectedSpan(widget._initialSelectedToken!.text)
: null;
/////////////////////////////////////
/// State setting
@ -250,19 +243,6 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
?.firstWhereOrNull(isTokenSelected);
}
bool get showLanguageAssistance {
if (!event.status.isSent || event.type != EventTypes.Message) {
return false;
}
if (event.messageType == MessageTypes.Text) {
return pangeaMessageEvent.messageDisplayLangCode.split("-").first ==
MatrixState.pangeaController.languageController.userL2!.langCodeShort;
}
return event.messageType == MessageTypes.Audio;
}
/// If sentence TTS is playing a word, highlight that word in message overlay
void highlightCurrentText(int currentPosition, List<TTSToken> ttsTokens) {
final List<TTSToken> textToSelect = [];
@ -302,22 +282,8 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
void onClickOverlayMessageToken(
PangeaToken token,
) {
// /// we don't want to associate the audio with the text in this mode
// if (practiceSelection?.hasActiveActivityByToken(
// ActivityTypeEnum.wordFocusListening,
// token,
// ) ==
// false ||
// !hideWordCardContent) {
// TtsController.tryToSpeak(
// token.text.content,
// targetID: null,
// langCode: pangeaMessageEvent.messageDisplayLangCode,
// );
// }
updateSelectedSpan(token.text);
}
) =>
updateSelectedSpan(token.text);
/// Whether the given token is currently selected or highlighted
bool isTokenSelected(PangeaToken token) {

2
lib/pangea/toolbar/widgets/over_message_overlay.dart
View file

@ -58,7 +58,7 @@ class OverMessageOverlay extends StatelessWidget {
? controller.originalMessageSize.height
: null,
messageWidth: controller.widget.overlayController
.selectModeController.showingExtraContent
.selectModeController.isShowingExtraContent
? max(controller.originalMessageSize.width, 150)
: controller.originalMessageSize.width,
overlayController: controller.widget.overlayController,

19
lib/pangea/toolbar/widgets/select_mode_buttons.dart
View file

@ -146,17 +146,6 @@ class SelectModeButtonsState extends State<SelectModeButtons> {
StreamSubscription? _playerStateSub;
StreamSubscription? _audioSub;
static List<SelectMode> get textModes => [
SelectMode.audio,
SelectMode.translate,
SelectMode.practice,
SelectMode.emoji,
];
static List<SelectMode> get audioModes => [
SelectMode.speechTranslation,
];
MatrixState? matrix;
@override
@ -309,13 +298,7 @@ class SelectModeButtonsState extends State<SelectModeButtons> {
@override
Widget build(BuildContext context) {
final theme = Theme.of(context);
final List<SelectMode> modes =
widget.overlayController.showLanguageAssistance
? messageEvent.isAudioMessage == true
? audioModes
: textModes
: [];
final modes = controller.readingAssistanceModes;
return Material(
type: MaterialType.transparency,
child: SizedBox(

398
lib/pangea/toolbar/widgets/select_mode_controller.dart
View file

@ -7,7 +7,6 @@ import 'package:matrix/matrix.dart';
import 'package:path_provider/path_provider.dart';
import 'package:fluffychat/pangea/common/utils/async_state.dart';
import 'package:fluffychat/pangea/common/utils/error_handler.dart';
import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart';
import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart';
import 'package:fluffychat/pangea/toolbar/models/speech_to_text_models.dart';
@ -15,270 +14,173 @@ import 'package:fluffychat/pangea/toolbar/widgets/message_audio_card.dart';
import 'package:fluffychat/pangea/toolbar/widgets/select_mode_buttons.dart';
import 'package:fluffychat/widgets/matrix.dart';
class _TranscriptionLoader extends AsyncLoader<SpeechToTextModel> {
final PangeaMessageEvent messageEvent;
_TranscriptionLoader(this.messageEvent) : super();
@override
Future<SpeechToTextModel> fetch() => messageEvent.getSpeechToText(
MatrixState.pangeaController.languageController.userL1!.langCodeShort,
MatrixState.pangeaController.languageController.userL2!.langCodeShort,
);
}
class _STTTranslationLoader extends AsyncLoader<String> {
final PangeaMessageEvent messageEvent;
_STTTranslationLoader(this.messageEvent) : super();
@override
Future<String> fetch() => messageEvent.sttTranslationByLanguageGlobal(
langCode: MatrixState
.pangeaController.languageController.userL1!.langCodeShort,
l1Code: MatrixState
.pangeaController.languageController.userL1!.langCodeShort,
l2Code: MatrixState
.pangeaController.languageController.userL2!.langCodeShort,
);
}
class _TranslationLoader extends AsyncLoader<String> {
final PangeaMessageEvent messageEvent;
_TranslationLoader(this.messageEvent) : super();
@override
Future<String> fetch() => messageEvent.l1Respresentation();
}
class _AudioLoader extends AsyncLoader<(PangeaAudioFile, File?)> {
final PangeaMessageEvent messageEvent;
_AudioLoader(this.messageEvent) : super();
@override
Future<(PangeaAudioFile, File?)> fetch() async {
final String langCode = messageEvent.messageDisplayLangCode;
final Event? localEvent = messageEvent.getTextToSpeechLocal(
langCode,
messageEvent.messageDisplayText,
);
PangeaAudioFile? audioBytes;
if (localEvent != null) {
audioBytes = await localEvent.getPangeaAudioFile();
} else {
audioBytes = await messageEvent.getMatrixAudioFile(
langCode,
);
}
if (audioBytes == null) {
throw Exception('Audio bytes are null');
}
File? audioFile;
if (!kIsWeb) {
final tempDir = await getTemporaryDirectory();
File? file;
file = File('${tempDir.path}/${audioBytes.name}');
await file.writeAsBytes(audioBytes.bytes);
audioFile = file;
}
return (audioBytes, audioFile);
}
}
class SelectModeController {
final PangeaMessageEvent messageEvent;
final _TranscriptionLoader _transcriptLoader;
final _TranslationLoader _translationLoader;
final _AudioLoader _audioLoader;
final _STTTranslationLoader _sttTranslationLoader;
SelectModeController(
this.messageEvent,
);
) : _transcriptLoader = _TranscriptionLoader(messageEvent),
_translationLoader = _TranslationLoader(messageEvent),
_audioLoader = _AudioLoader(messageEvent),
_sttTranslationLoader = _STTTranslationLoader(messageEvent);
ValueNotifier<SelectMode?> selectedMode = ValueNotifier<SelectMode?>(null);
final ValueNotifier<AsyncState<SpeechToTextModel>> transcriptionState =
ValueNotifier<AsyncState<SpeechToTextModel>>(const AsyncState.idle());
final ValueNotifier<AsyncState<String>> translationState =
ValueNotifier<AsyncState<String>>(const AsyncState.idle());
final ValueNotifier<AsyncState<String>> speechTranslationState =
ValueNotifier<AsyncState<String>>(const AsyncState.idle());
final ValueNotifier<AsyncState<(PangeaAudioFile, File?)>> audioState =
ValueNotifier<AsyncState<(PangeaAudioFile, File?)>>(
const AsyncState.idle(),
);
final StreamController contentChangedStream = StreamController.broadcast();
bool _disposed = false;
bool get showingExtraContent =>
(selectedMode.value == SelectMode.translate &&
translationState.value is AsyncLoaded) ||
(selectedMode.value == SelectMode.speechTranslation &&
speechTranslationState.value is AsyncLoaded) ||
transcriptionState.value is AsyncLoaded ||
transcriptionState.value is AsyncError;
String? get l1Code =>
MatrixState.pangeaController.languageController.userL1?.langCodeShort;
String? get l2Code =>
MatrixState.pangeaController.languageController.userL2?.langCodeShort;
(PangeaAudioFile, File?)? get audioFile => audioState.value is AsyncLoaded
? (audioState.value as AsyncLoaded<(PangeaAudioFile, File?)>).value
: null;
ValueNotifier<AsyncState>? modeStateNotifier(SelectMode mode) {
switch (mode) {
case SelectMode.audio:
return audioState;
case SelectMode.translate:
return translationState;
case SelectMode.speechTranslation:
return speechTranslationState;
default:
return null;
}
}
ValueNotifier<AsyncState>? get currentModeStateNotifier {
final mode = selectedMode.value;
if (mode == null) return null;
return modeStateNotifier(mode);
}
void dispose() {
selectedMode.dispose();
transcriptionState.dispose();
translationState.dispose();
speechTranslationState.dispose();
audioState.dispose();
_transcriptLoader.dispose();
_translationLoader.dispose();
_sttTranslationLoader.dispose();
_audioLoader.dispose();
contentChangedStream.close();
_disposed = true;
}
static List<SelectMode> get textModes => [
SelectMode.audio,
SelectMode.translate,
SelectMode.practice,
SelectMode.emoji,
];
static List<SelectMode> get audioModes => [
SelectMode.speechTranslation,
];
ValueNotifier<AsyncState<String>> get translationState =>
_translationLoader.state;
ValueNotifier<AsyncState<SpeechToTextModel>> get transcriptionState =>
_transcriptLoader.state;
ValueNotifier<AsyncState<String>> get speechTranslationState =>
_sttTranslationLoader.state;
(PangeaAudioFile, File?)? get audioFile => _audioLoader.value;
List<SelectMode> get readingAssistanceModes {
final validTypes = {MessageTypes.Text, MessageTypes.Audio};
if (!messageEvent.event.status.isSent ||
messageEvent.event.type != EventTypes.Message ||
!validTypes.contains(messageEvent.event.messageType)) {
return [];
}
if (messageEvent.event.messageType == MessageTypes.Text) {
final matchesL2 = messageEvent.messageDisplayLangCode.split("-").first ==
MatrixState.pangeaController.languageController.userL2!.langCodeShort;
return matchesL2 ? textModes : [SelectMode.translate];
}
return audioModes;
}
bool get isLoading => currentModeStateNotifier?.value is AsyncLoading;
bool get isShowingExtraContent =>
(selectedMode.value == SelectMode.translate &&
_translationLoader.isLoaded) ||
(selectedMode.value == SelectMode.speechTranslation &&
_sttTranslationLoader.isLoaded) ||
_transcriptLoader.isLoaded ||
_transcriptLoader.isError;
ValueNotifier<AsyncState>? get currentModeStateNotifier =>
modeStateNotifier(selectedMode.value);
ValueNotifier<AsyncState>? modeStateNotifier(SelectMode? mode) =>
switch (mode) {
SelectMode.audio => _audioLoader.state,
SelectMode.translate => _translationLoader.state,
SelectMode.speechTranslation => _sttTranslationLoader.state,
_ => null,
};
void setSelectMode(SelectMode? mode) {
if (selectedMode.value == mode) return;
selectedMode.value = mode;
}
Future<void> fetchAudio() async {
audioState.value = const AsyncState.loading();
try {
final String langCode = messageEvent.messageDisplayLangCode;
final Event? localEvent = messageEvent.getTextToSpeechLocal(
langCode,
messageEvent.messageDisplayText,
);
PangeaAudioFile? audioBytes;
if (localEvent != null) {
audioBytes = await localEvent.getPangeaAudioFile();
} else {
audioBytes = await messageEvent.getMatrixAudioFile(
langCode,
);
}
if (_disposed) return;
if (audioBytes == null) {
throw Exception('Audio bytes are null');
}
File? audioFile;
if (!kIsWeb) {
final tempDir = await getTemporaryDirectory();
File? file;
file = File('${tempDir.path}/${audioBytes.name}');
await file.writeAsBytes(audioBytes.bytes);
audioFile = file;
}
audioState.value = AsyncState.loaded((audioBytes, audioFile));
} catch (e, s) {
ErrorHandler.logError(
e: e,
s: s,
m: 'something wrong getting audio in MessageAudioCardState',
data: {
'widget.messageEvent.messageDisplayLangCode':
messageEvent.messageDisplayLangCode,
},
);
if (_disposed) return;
audioState.value = AsyncState.error(e);
}
}
Future<void> fetchTranslation() async {
if (l1Code == null ||
translationState.value is AsyncLoading ||
translationState.value is AsyncLoaded) {
return;
}
try {
translationState.value = const AsyncState.loading();
final rep = await messageEvent.l1Respresentation();
if (_disposed) return;
translationState.value = AsyncState.loaded(rep.text);
} catch (e, s) {
ErrorHandler.logError(
e: e,
s: s,
m: 'Error fetching translation',
data: {
'l1Code': l1Code,
'messageEvent': messageEvent.event.toJson(),
},
);
if (_disposed) return;
translationState.value = AsyncState.error(e);
}
}
Future<void> fetchTranscription() async {
try {
if (transcriptionState.value is AsyncLoading ||
transcriptionState.value is AsyncLoaded) {
// If a transcription is already in progress or finished, don't fetch again
return;
}
if (l1Code == null || l2Code == null) {
transcriptionState.value = const AsyncState.error(
'Language code or message event is null',
);
return;
}
final resp = await messageEvent.getSpeechToText(
l1Code!,
l2Code!,
);
if (_disposed) return;
if (resp == null) {
transcriptionState.value = const AsyncState.error(
'Transcription response is null',
);
return;
}
transcriptionState.value = AsyncState.loaded(resp);
} catch (err, s) {
ErrorHandler.logError(
e: err,
s: s,
data: {},
);
if (_disposed) return;
transcriptionState.value = AsyncState.error(err);
}
}
Future<void> fetchSpeechTranslation() async {
if (l1Code == null ||
l2Code == null ||
speechTranslationState.value is AsyncLoading ||
speechTranslationState.value is AsyncLoaded) {
return;
}
if (transcriptionState.value is AsyncError) {
speechTranslationState.value = AsyncState.error(
(transcriptionState.value as AsyncError).error,
);
return;
}
try {
speechTranslationState.value = const AsyncState.loading();
if (transcriptionState.value is AsyncIdle ||
transcriptionState.value is AsyncLoading) {
await fetchTranscription();
if (_disposed) return;
if (transcriptionState.value is! AsyncLoaded) {
throw Exception('Transcription is null');
}
}
final translation = await messageEvent.sttTranslationByLanguageGlobal(
langCode: l1Code!,
l1Code: l1Code!,
l2Code: l2Code!,
);
if (translation == null) {
throw Exception('Translation is null');
}
if (_disposed) return;
speechTranslationState.value = AsyncState.loaded(translation.translation);
} catch (err, s) {
ErrorHandler.logError(
e: err,
s: s,
data: {},
);
if (_disposed) return;
speechTranslationState.value = AsyncState.error(err);
}
}
bool get isError {
switch (selectedMode.value) {
case SelectMode.audio:
return audioState.value is AsyncError;
case SelectMode.translate:
return translationState.value is AsyncError;
case SelectMode.speechTranslation:
return speechTranslationState.value is AsyncError;
default:
return false;
}
}
bool get isLoading {
switch (selectedMode.value) {
case SelectMode.audio:
return audioState.value is AsyncLoading;
case SelectMode.translate:
return translationState.value is AsyncLoading;
case SelectMode.speechTranslation:
return speechTranslationState.value is AsyncLoading;
default:
return false;
}
}
Future<void> fetchAudio() => _audioLoader.load();
Future<void> fetchTranslation() => _translationLoader.load();
Future<void> fetchTranscription() => _transcriptLoader.load();
Future<void> fetchSpeechTranslation() => _sttTranslationLoader.load();
}