ember/fluffychat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore: require passing of langCode to TTS, log langCode (#2529)

Browse source
This commit is contained in:
ggurdin 2025-04-22 13:42:45 -04:00 • committed by GitHub
parent 9b547d702b
commit fe88836e89
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 185 additions and 139 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
lib/pangea/analytics_details_popup/morph_meaning_widget.dart
View file

@ -1,13 +1,15 @@
import 'dart:developer';
import 'dart:math';
import 'package:flutter/foundation.dart';
import 'package:flutter/material.dart';
import 'package:flutter_gen/gen_l10n/l10n.dart';
import 'package:fluffychat/pangea/analytics_misc/text_loading_shimmer.dart';
import 'package:fluffychat/pangea/morphs/get_grammar_copy.dart';
import 'package:fluffychat/pangea/morphs/morph_features_enum.dart';
import 'package:fluffychat/pangea/morphs/morph_meaning/morph_info_repo.dart';
import 'package:flutter/foundation.dart';
import 'package:flutter/material.dart';
import 'package:flutter_gen/gen_l10n/l10n.dart';
class MorphMeaningWidget extends StatefulWidget {
final MorphFeaturesEnum feature;

9
lib/pangea/choreographer/widgets/choice_array.dart
View file

@ -35,6 +35,9 @@ class ChoicesArray extends StatefulWidget {
final bool enableAudio;
/// language code for the TTS
final String? langCode;
/// Used to unqiuely identify the keys for choices, in cases where multiple
/// choices could have identical text, like in back-to-back practice activities
final String? id;
@ -61,6 +64,7 @@ class ChoicesArray extends StatefulWidget {
required this.selectedChoiceIndex,
required this.tts,
this.enableAudio = true,
this.langCode,
this.isActive = true,
this.onLongPress,
this.getDisplayCopy,
@ -107,11 +111,14 @@ class ChoicesArrayState extends State<ChoicesArray> {
? (String value, int index) {
widget.onPressed(value, index);
// TODO - what to pass here as eventID?
if (widget.enableAudio && widget.tts != null) {
if (widget.enableAudio &&
widget.tts != null &&
widget.langCode != null) {
widget.tts?.tryToSpeak(
value,
context,
targetID: null,
langCode: widget.langCode!,
);
}
}

2
lib/pangea/choreographer/widgets/igc/span_card.dart
View file

@ -310,6 +310,8 @@ class WordMatchContent extends StatelessWidget {
tts: controller.tts,
id: controller.widget.scm.pangeaMatch!.hashCode
.toString(),
langCode: MatrixState.pangeaController.languageController
.activeL2Code(),
),
const SizedBox(height: 12),
PromptAndFeedback(controller: controller),

2
lib/pangea/choreographer/widgets/it_bar.dart
View file

@ -444,6 +444,8 @@ class ITChoices extends StatelessWidget {
onLongPress: (value, index) => showCard(context, index),
selectedChoiceIndex: null,
tts: controller.choreographer.tts,
langCode: controller.choreographer.pangeaController.languageController
.activeL2Code(),
);
} catch (e) {
debugger(when: kDebugMode);

2
lib/pangea/learning_settings/pages/settings_learning.dart
View file

@ -44,7 +44,7 @@ class SettingsLearningController extends State<SettingsLearning> {
void initState() {
super.initState();
_profile = pangeaController.userController.profile.copy();
tts.setupTTS().then((_) => setState(() {}));
tts.setAvailableLanguages().then((_) => setState(() {}));
}
@override

218
lib/pangea/toolbar/controllers/tts_controller.dart
View file

@ -8,7 +8,8 @@ import 'package:flutter/material.dart';
import 'package:collection/collection.dart';
import 'package:flutter_tts/flutter_tts.dart' as flutter_tts;
import 'package:just_audio/just_audio.dart';
import 'package:matrix/matrix_api_lite/utils/logs.dart';
import 'package:matrix/matrix.dart';
import 'package:sentry_flutter/sentry_flutter.dart';
import 'package:text_to_speech/text_to_speech.dart';
import 'package:fluffychat/pages/chat/chat.dart';
@ -18,35 +19,27 @@ import 'package:fluffychat/pangea/instructions/instructions_enum.dart';
import 'package:fluffychat/pangea/instructions/instructions_show_popup.dart';
import 'package:fluffychat/pangea/learning_settings/constants/language_constants.dart';
import 'package:fluffychat/pangea/toolbar/controllers/text_to_speech_controller.dart';
import 'package:fluffychat/pangea/user/controllers/user_controller.dart';
import 'package:fluffychat/utils/platform_infos.dart';
import 'package:fluffychat/widgets/matrix.dart';
class TtsController {
final ChatController? chatController;
String? get l2LangCode =>
MatrixState.pangeaController.languageController.userL2?.langCode;
String? get l2LangCodeShort =>
MatrixState.pangeaController.languageController.userL2?.langCodeShort;
TtsController({this.chatController}) {
setAvailableLanguages();
_languageSubscription =
MatrixState.pangeaController.userController.stateStream.listen(
(_) => setAvailableLanguages(),
);
}
List<String> _availableLangCodes = [];
StreamSubscription? _languageSubscription;
final flutter_tts.FlutterTts _tts = flutter_tts.FlutterTts();
final TextToSpeech _alternativeTTS = TextToSpeech();
StreamSubscription? _languageSubscription;
final StreamController<bool> loadingChoreoStream =
StreamController<bool>.broadcast();
UserController get userController =>
MatrixState.pangeaController.userController;
TtsController({this.chatController}) {
setupTTS();
_languageSubscription =
userController.stateStream.listen((_) => setupTTS());
}
bool get _useAlternativeTTS {
return PlatformInfos.isWindows;
}
@ -72,7 +65,27 @@ class TtsController {
);
}
Future<void> _setAvailableLanguages() async {
Future<void> setAvailableLanguages() async {
try {
if (_useAlternativeTTS) {
await _setAvailableAltLanguages();
} else {
_tts.setErrorHandler(_onError);
await _tts.awaitSpeakCompletion(true);
await _setAvailableBaseLanguages();
}
} catch (e, s) {
debugger(when: kDebugMode);
ErrorHandler.logError(
e: e,
s: s,
data: {},
);
}
}
Future<void> _setAvailableBaseLanguages() async {
final voices = (await _tts.getVoices) as List?;
_availableLangCodes = (voices ?? [])
.map((v) {
@ -91,51 +104,29 @@ class TtsController {
_availableLangCodes = languages.toSet().toList();
}
Future<void> _setLanguage(String? langCode) async {
Future<void> _setSpeakingLanguage(String langCode) async {
String? selectedLangCode;
if (langCode != null) {
final langCodeShort = langCode.split("-").first;
if (_availableLangCodes.contains(langCode)) {
selectedLangCode = langCode;
} else {
selectedLangCode = _availableLangCodes.firstWhereOrNull(
(code) => code.startsWith(langCodeShort),
);
}
final langCodeShort = langCode.split("-").first;
if (_availableLangCodes.contains(langCode)) {
selectedLangCode = langCode;
} else {
if (_availableLangCodes.contains(l2LangCode)) {
selectedLangCode = l2LangCode;
} else if (l2LangCodeShort != null) {
selectedLangCode = _availableLangCodes.firstWhereOrNull(
(code) => code.startsWith(l2LangCodeShort!),
);
}
selectedLangCode = _availableLangCodes.firstWhereOrNull(
(code) => code.startsWith(langCodeShort),
);
}
if (selectedLangCode != null) {
await (_useAlternativeTTS
? _alternativeTTS.setLanguage(selectedLangCode)
: _tts.setLanguage(selectedLangCode));
}
}
Future<void> setupTTS() async {
try {
if (_useAlternativeTTS) {
await _setAvailableAltLanguages();
} else {
_tts.setErrorHandler(_onError);
debugger(when: kDebugMode && l2LangCode == null);
await _tts.awaitSpeakCompletion(true);
await _setAvailableLanguages();
}
} catch (e, s) {
debugger(when: kDebugMode);
ErrorHandler.logError(
e: e,
s: s,
data: {},
} else {
final jsonData = {
'langCode': langCode,
'availableLangCodes': _availableLangCodes,
};
debugPrint("TTS: Language not supported: $jsonData");
Sentry.addBreadcrumb(
Breadcrumb.fromJson(jsonData),
);
}
}
@ -165,48 +156,41 @@ class TtsController {
}
}
Future<void> _showTTSDisabledPopup(
BuildContext context,
String targetID,
) async =>
instructionsShowPopup(
context,
InstructionsEnum.ttsDisabled,
targetID,
showToggle: false,
forceShow: true,
);
/// A safer version of speak, that handles the case of
/// the language not being supported by the TTS engine
Future<void> tryToSpeak(
String text,
BuildContext context, {
required String langCode,
// Target ID for where to show warning popup
String? targetID,
String? langCode,
}) async {
chatController?.stopAudioStream.add(null);
await _setLanguage(langCode);
await _setSpeakingLanguage(langCode);
final enableTTS = MatrixState
.pangeaController.userController.profile.toolSettings.enableTTS;
if (enableTTS) {
if (_isL2FullySupported) {
await _speak(text);
} else {
await _speakFromChoreo(text);
}
await (_isLangFullySupported(langCode)
? _speak(
text,
langCode,
)
: _speakFromChoreo(
text,
langCode,
));
} else if (targetID != null) {
await _showTTSDisabledPopup(context, targetID);
}
}
Future<void> _speak(String text) async {
Future<void> _speak(String text, String langCode) async {
try {
stop();
text = text.toLowerCase();
Logs().i('Speaking: $text');
Logs().i('Speaking: $text, langCode: $langCode');
final result = await Future(
() => (_useAlternativeTTS
? _alternativeTTS.speak(text)
@ -244,44 +228,28 @@ class TtsController {
'text': text,
},
);
await _speakFromChoreo(text);
await _speakFromChoreo(text, langCode);
}
}
Future<void> playAudio(Uint8List audioContent, String mimeType) async {
final audioPlayer = AudioPlayer();
try {
await audioPlayer
.setAudioSource(BytesAudioSource(audioContent, mimeType));
await audioPlayer.play();
} catch (e) {
ErrorHandler.logError(
e: 'Error playing audio',
data: {
'error': e.toString(),
},
);
}
}
bool get _isL2FullySupported {
return _availableLangCodes.contains(l2LangCode) ||
(l2LangCodeShort != null &&
_availableLangCodes
.any((lang) => lang.startsWith(l2LangCodeShort!)));
}
Future<void> _speakFromChoreo(String text) async {
Future<void> _speakFromChoreo(
String text,
String langCode,
) async {
TextToSpeechResponse? ttsRes;
try {
loadingChoreoStream.add(true);
ttsRes = await chatController?.pangeaController.textToSpeech.get(
TextToSpeechRequest(
text: text,
langCode: l2LangCode ?? LanguageKeys.unknownLanguage,
langCode: langCode,
tokens: [], // TODO: Somehow bring existing tokens to avoid extra choreo token requests
userL1: LanguageKeys.unknownLanguage,
userL2: LanguageKeys.unknownLanguage,
userL1:
MatrixState.pangeaController.languageController.activeL1Code() ??
LanguageKeys.unknownLanguage,
userL2:
MatrixState.pangeaController.languageController.activeL2Code() ??
LanguageKeys.unknownLanguage,
),
);
} catch (e, s) {
@ -298,17 +266,53 @@ class TtsController {
if (ttsRes == null) return;
final audioPlayer = AudioPlayer();
try {
Logs().i('Speaking from choreo: $text, langCode: $langCode');
final audioContent = base64Decode(ttsRes.audioContent);
await playAudio(audioContent, ttsRes.mimeType);
await audioPlayer.setAudioSource(
BytesAudioSource(
audioContent,
ttsRes.mimeType,
),
);
await audioPlayer.play();
} catch (e, s) {
ErrorHandler.logError(
e: e,
e: 'Error playing audio',
s: s,
data: {
'error': e.toString(),
'text': text,
},
);
} finally {
await audioPlayer.dispose();
}
}
bool _isLangFullySupported(String langCode) {
if (_availableLangCodes.contains(langCode)) {
return true;
}
final langCodeShort = langCode.split("-").first;
if (langCodeShort.isEmpty) {
return false;
}
return _availableLangCodes.any((lang) => lang.startsWith(langCodeShort));
}
Future<void> _showTTSDisabledPopup(
BuildContext context,
String targetID,
) async =>
instructionsShowPopup(
context,
InstructionsEnum.ttsDisabled,
targetID,
showToggle: false,
forceShow: true,
);
}

6
lib/pangea/toolbar/reading_assistance_input_row/message_morph_choice.dart
View file

@ -1,4 +1,8 @@
import 'package:flutter/material.dart';
import 'package:collection/collection.dart';
import 'package:flutter_gen/gen_l10n/l10n.dart';
import 'package:fluffychat/pangea/analytics_details_popup/morph_meaning_widget.dart';
import 'package:fluffychat/pangea/analytics_misc/construct_type_enum.dart';
import 'package:fluffychat/pangea/choreographer/widgets/choice_animation.dart';
@ -12,8 +16,6 @@ import 'package:fluffychat/pangea/practice_activities/practice_activity_model.da
import 'package:fluffychat/pangea/practice_activities/practice_choice.dart';
import 'package:fluffychat/pangea/toolbar/reading_assistance_input_row/message_morph_choice_item.dart';
import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart';
import 'package:flutter/material.dart';
import 'package:flutter_gen/gen_l10n/l10n.dart';
// this widget will handle the content of the input bar when mode == MessageMode.wordMorph

3
lib/pangea/toolbar/reading_assistance_input_row/message_morph_choice_item.dart
View file

@ -1,10 +1,11 @@
import 'package:flutter/material.dart';
import 'package:fluffychat/config/app_config.dart';
import 'package:fluffychat/config/themes.dart';
import 'package:fluffychat/pangea/constructs/construct_identifier.dart';
import 'package:fluffychat/pangea/morphs/get_grammar_copy.dart';
import 'package:fluffychat/pangea/morphs/morph_features_enum.dart';
import 'package:fluffychat/pangea/morphs/morph_icon.dart';
import 'package:flutter/material.dart';
class MessageMorphChoiceItem extends StatefulWidget {
const MessageMorphChoiceItem({

21
lib/pangea/toolbar/reading_assistance_input_row/practice_match_item.dart
View file

@ -1,13 +1,15 @@
import 'dart:developer';
import 'package:flutter/foundation.dart';
import 'package:flutter/material.dart';
import 'package:fluffychat/config/app_config.dart';
import 'package:fluffychat/pangea/common/utils/error_handler.dart';
import 'package:fluffychat/pangea/events/models/pangea_token_model.dart';
import 'package:fluffychat/pangea/practice_activities/practice_choice.dart';
import 'package:fluffychat/pangea/toolbar/controllers/tts_controller.dart';
import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart';
import 'package:flutter/foundation.dart';
import 'package:flutter/material.dart';
import 'package:fluffychat/widgets/matrix.dart';
class PracticeMatchItem extends StatefulWidget {
const PracticeMatchItem({
@ -61,11 +63,16 @@ class PracticeMatchItemState extends State<PracticeMatchItem> {
setState(() => _isPlaying = true);
}
try {
await tts.tryToSpeak(
widget.audioContent!,
context,
targetID: 'word-audio-button',
);
final l2 =
MatrixState.pangeaController.languageController.activeL2Code();
if (l2 != null) {
await tts.tryToSpeak(
widget.audioContent!,
context,
targetID: 'word-audio-button',
langCode: l2,
);
}
} catch (e, s) {
debugger(when: kDebugMode);
ErrorHandler.logError(

13
lib/pangea/toolbar/widgets/message_selection_overlay.dart
View file

@ -537,17 +537,18 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
}
/// we don't want to associate the audio with the text in this mode
if (practiceSelection?.hasActiveActivityByToken(
ActivityTypeEnum.wordFocusListening,
token,
) ==
false ||
if (pangeaMessageEvent?.messageDisplayLangCode != null &&
practiceSelection?.hasActiveActivityByToken(
ActivityTypeEnum.wordFocusListening,
token,
) ==
false ||
!hideWordCardContent) {
widget.chatController.choreographer.tts.tryToSpeak(
token.text.content,
context,
targetID: null,
langCode: pangeaMessageEvent?.messageDisplayLangCode,
langCode: pangeaMessageEvent!.messageDisplayLangCode,
);
}

4
lib/pangea/toolbar/widgets/practice_activity/multiple_choice_activity.dart
View file

@ -231,6 +231,8 @@ class MultipleChoiceActivityState extends State<MultipleChoiceActivity> {
WordAudioButton(
text: practiceActivity.multipleChoiceContent!.answers.first,
uniqueID: "audio-activity-${widget.event.eventId}",
langCode: widget
.overlayController.pangeaMessageEvent?.messageDisplayLangCode,
),
if (practiceActivity.activityType ==
ActivityTypeEnum.hiddenWordListening)
@ -251,6 +253,8 @@ class MultipleChoiceActivityState extends State<MultipleChoiceActivity> {
id: currentRecordModel?.hashCode.toString(),
tts: practiceActivity.activityType.includeTTSOnClick ? tts : null,
enableAudio: !widget.overlayController.isPlayingAudio,
langCode:
MatrixState.pangeaController.languageController.activeL2Code(),
getDisplayCopy: _getDisplayCopy,
enableMultiSelect:
widget.currentActivity.activityType == ActivityTypeEnum.emoji,

15
lib/pangea/toolbar/widgets/practice_activity/word_audio_button.dart
View file

@ -12,6 +12,7 @@ class WordAudioButton extends StatefulWidget {
final bool isSelected;
final double baseOpacity;
final String uniqueID;
final String? langCode;
/// If defined, this callback will be called instead of the default one
final void Function()? callbackOverride;
@ -24,6 +25,7 @@ class WordAudioButton extends StatefulWidget {
this.isSelected = false,
this.baseOpacity = 1,
this.callbackOverride,
this.langCode,
});
@override
@ -82,11 +84,14 @@ class WordAudioButtonState extends State<WordAudioButton> {
setState(() => _isPlaying = true);
}
try {
await tts.tryToSpeak(
widget.text,
context,
targetID: 'word-audio-button-${widget.uniqueID}',
);
if (widget.langCode != null) {
await tts.tryToSpeak(
widget.text,
context,
targetID: 'word-audio-button-${widget.uniqueID}',
langCode: widget.langCode!,
);
}
} catch (e, s) {
ErrorHandler.logError(
e: e,

15
lib/pangea/toolbar/widgets/practice_activity/word_text_with_audio_button.dart
View file

@ -78,11 +78,16 @@ class WordAudioButtonState extends State<WordTextWithAudioButton> {
setState(() => _isPlaying = true);
}
try {
await tts.tryToSpeak(
widget.text,
context,
targetID: 'text-audio-button-${widget.uniqueID}',
);
final l2 = MatrixState.pangeaController.languageController
.activeL2Code();
if (l2 != null) {
await tts.tryToSpeak(
widget.text,
context,
targetID: 'text-audio-button-${widget.uniqueID}',
langCode: l2,
);
}
} catch (e, s) {
ErrorHandler.logError(
e: e,

3
lib/pangea/toolbar/widgets/toolbar_button_column.dart
View file

@ -1,7 +1,8 @@
import 'package:flutter/material.dart';
import 'package:fluffychat/pangea/toolbar/enums/message_mode_enum.dart';
import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart';
import 'package:fluffychat/pangea/toolbar/widgets/toolbar_button.dart';
import 'package:flutter/material.dart';
class ToolbarButtonRow extends StatelessWidget {
final MessageOverlayController overlayController;

1
lib/pangea/toolbar/widgets/word_zoom/lemma_widget.dart
View file

@ -201,6 +201,7 @@ class LemmaWidgetState extends State<LemmaWidget> {
?.updateToolbarMode(MessageMode.listening)
: null,
uniqueID: "lemma-content-${widget.token.text.content}",
langCode: widget.pangeaMessageEvent.messageDisplayLangCode,
),
],
);

2
lib/pangea/toolbar/widgets/word_zoom/word_zoom_widget.dart
View file

@ -184,6 +184,8 @@ class WordZoomWidget extends StatelessWidget {
.updateToolbarMode(MessageMode.listening)
: null,
uniqueID: "word-zoom-audio-${_selectedToken.text.content}",
langCode: overlayController
.pangeaMessageEvent?.messageDisplayLangCode,
),
],
..._selectedToken.morphsBasicallyEligibleForPracticeByPriority