From 74b0cfd584089940e7b5503fea25c3bac48787b0 Mon Sep 17 00:00:00 2001 From: Wilson Date: Mon, 10 Mar 2025 10:14:11 -0400 Subject: [PATCH] feat: text to speech for all (#2090) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: ggurdin --- .../pages/settings_learning.dart | 5 +- .../pages/settings_learning_view.dart | 62 ------------ .../text_to_speech_controller.dart | 2 +- .../toolbar/controllers/tts_controller.dart | 96 +++++++++++++------ .../word_text_with_audio_button.dart | 45 ++++++++- 5 files changed, 111 insertions(+), 99 deletions(-) diff --git a/lib/pangea/learning_settings/pages/settings_learning.dart b/lib/pangea/learning_settings/pages/settings_learning.dart index 88b477891..9c724dda8 100644 --- a/lib/pangea/learning_settings/pages/settings_learning.dart +++ b/lib/pangea/learning_settings/pages/settings_learning.dart @@ -193,7 +193,6 @@ class SettingsLearningController extends State { case ToolSetting.enableTTS: return _profile.userSettings.targetLanguage != null && _targetLanguage != null && - tts.isLanguageSupported(_targetLanguage!) && toolSettings.enableTTS; case ToolSetting.enableAutocorrect: return toolSettings.enableAutocorrect; @@ -201,9 +200,7 @@ class SettingsLearningController extends State { } bool get isTTSSupported => - _profile.userSettings.targetLanguage != null && - _targetLanguage != null && - tts.isLanguageSupported(_targetLanguage!); + _profile.userSettings.targetLanguage != null && _targetLanguage != null; LanguageModel? get selectedSourceLanguage { return userL1 ?? pangeaController.languageController.systemLanguage; diff --git a/lib/pangea/learning_settings/pages/settings_learning_view.dart b/lib/pangea/learning_settings/pages/settings_learning_view.dart index 970a910c2..f872a9065 100644 --- a/lib/pangea/learning_settings/pages/settings_learning_view.dart +++ b/lib/pangea/learning_settings/pages/settings_learning_view.dart @@ -1,7 +1,6 @@ import 'dart:io'; import 'package:flutter/foundation.dart'; -import 'package:flutter/gestures.dart'; import 'package:flutter/material.dart'; import 'package:app_settings/app_settings.dart'; @@ -18,7 +17,6 @@ import 'package:fluffychat/pangea/learning_settings/widgets/country_picker_tile. import 'package:fluffychat/pangea/learning_settings/widgets/p_language_dropdown.dart'; import 'package:fluffychat/pangea/learning_settings/widgets/p_settings_switch_list_tile.dart'; import 'package:fluffychat/pangea/spaces/models/space_model.dart'; -import 'package:fluffychat/utils/platform_infos.dart'; import 'package:fluffychat/widgets/matrix.dart'; class SettingsLearningView extends StatelessWidget { @@ -242,67 +240,7 @@ class SettingsLearningView extends StatelessWidget { toolSetting, value, ), - enabled: - toolSetting == ToolSetting.enableTTS - ? controller.isTTSSupported - : true, ), - if (toolSetting == ToolSetting.enableTTS && - !controller.isTTSSupported) - Row( - children: [ - Padding( - padding: const EdgeInsets.only( - right: 16.0, - ), - child: Icon( - Icons.info_outlined, - color: Theme.of(context) - .disabledColor, - ), - ), - Flexible( - child: RichText( - text: TextSpan( - text: L10n.of(context) - .couldNotFindTTS, - style: TextStyle( - color: Theme.of(context) - .disabledColor, - ), - children: [ - if (PlatformInfos.isWindows || - PlatformInfos.isAndroid) - TextSpan( - text: L10n.of(context) - .ttsInstructionsHyperlink, - style: const TextStyle( - color: Colors.blue, - fontWeight: - FontWeight.bold, - decoration: - TextDecoration - .underline, - ), - recognizer: - TapGestureRecognizer() - ..onTap = () { - launchUrlString( - PlatformInfos - .isWindows - ? AppConfig - .windowsTTSDownloadInstructions - : AppConfig - .androidTTSDownloadInstructions, - ); - }, - ), - ], - ), - ), - ), - ], - ), ], ), SwitchListTile.adaptive( diff --git a/lib/pangea/toolbar/controllers/text_to_speech_controller.dart b/lib/pangea/toolbar/controllers/text_to_speech_controller.dart index dfc9cee17..54a9768ca 100644 --- a/lib/pangea/toolbar/controllers/text_to_speech_controller.dart +++ b/lib/pangea/toolbar/controllers/text_to_speech_controller.dart @@ -7,9 +7,9 @@ import 'package:http/http.dart'; import 'package:fluffychat/pangea/common/config/environment.dart'; import 'package:fluffychat/pangea/common/constants/model_keys.dart'; import 'package:fluffychat/pangea/common/controllers/pangea_controller.dart'; +import 'package:fluffychat/pangea/common/network/requests.dart'; import 'package:fluffychat/pangea/common/network/urls.dart'; import 'package:fluffychat/pangea/events/models/pangea_token_text_model.dart'; -import '../../common/network/requests.dart'; class PangeaAudioEventData { final String text; diff --git a/lib/pangea/toolbar/controllers/tts_controller.dart b/lib/pangea/toolbar/controllers/tts_controller.dart index 30082044e..3b1d2a624 100644 --- a/lib/pangea/toolbar/controllers/tts_controller.dart +++ b/lib/pangea/toolbar/controllers/tts_controller.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:convert'; import 'dart:developer'; import 'package:flutter/foundation.dart'; @@ -6,20 +7,22 @@ import 'package:flutter/material.dart'; import 'package:collection/collection.dart'; import 'package:flutter_tts/flutter_tts.dart' as flutter_tts; +import 'package:just_audio/just_audio.dart'; import 'package:matrix/matrix_api_lite/utils/logs.dart'; import 'package:text_to_speech/text_to_speech.dart'; import 'package:fluffychat/pages/chat/chat.dart'; +import 'package:fluffychat/pages/chat/events/audio_player.dart'; import 'package:fluffychat/pangea/common/utils/error_handler.dart'; import 'package:fluffychat/pangea/instructions/instructions_enum.dart'; import 'package:fluffychat/pangea/instructions/instructions_show_popup.dart'; -import 'package:fluffychat/pangea/learning_settings/models/language_model.dart'; -import 'package:fluffychat/pangea/toolbar/widgets/missing_voice_button.dart'; +import 'package:fluffychat/pangea/learning_settings/constants/language_constants.dart'; +import 'package:fluffychat/pangea/toolbar/controllers/text_to_speech_controller.dart'; import 'package:fluffychat/pangea/user/controllers/user_controller.dart'; import 'package:fluffychat/utils/platform_infos.dart'; import 'package:fluffychat/widgets/matrix.dart'; -class TtsController { +class TtsController extends ChangeNotifier { final ChatController? chatController; String? get l2LangCode => @@ -46,9 +49,20 @@ class TtsController { return PlatformInfos.isWindows; } + bool _hasLoadedTextToSpeech = false; + bool get hasLoadedTextToSpeech => _hasLoadedTextToSpeech; + set hasLoadedTextToSpeech(bool value) { + if (_hasLoadedTextToSpeech != value) { + _hasLoadedTextToSpeech = value; + notifyListeners(); + } + } + + @override Future dispose() async { await _tts.stop(); await _languageSubscription?.cancel(); + super.dispose(); } void _onError(dynamic message) { @@ -149,22 +163,6 @@ class TtsController { } } - Future _showMissingVoicePopup( - BuildContext context, - String targetID, - ) async => - instructionsShowPopup( - context, - InstructionsEnum.missingVoice, - targetID, - showToggle: false, - customContent: const Padding( - padding: EdgeInsets.only(top: 12), - child: MissingVoiceButton(), - ), - forceShow: true, - ); - Future _showTTSDisabledPopup( BuildContext context, String targetID, @@ -189,12 +187,13 @@ class TtsController { await _setLanguage(); final enableTTS = MatrixState .pangeaController.userController.profile.toolSettings.enableTTS; - - if (_isL2FullySupported && enableTTS) { - await _speak(text); - } else if (!_isL2FullySupported && targetID != null) { - await _showMissingVoicePopup(context, targetID); - } else if (!enableTTS && targetID != null) { + if (enableTTS) { + if (_isL2FullySupported) { + await _speak(text); + } else { + await _speakFromChoreo(text); + } + } else if (targetID != null) { await _showTTSDisabledPopup(context, targetID); } } @@ -242,6 +241,23 @@ class TtsController { 'text': text, }, ); + await _speakFromChoreo(text); + } + } + + Future playAudio(Uint8List audioContent, String mimeType) async { + final audioPlayer = AudioPlayer(); + try { + await audioPlayer + .setAudioSource(BytesAudioSource(audioContent, mimeType)); + await audioPlayer.play(); + } catch (e) { + ErrorHandler.logError( + e: 'Error playing audio', + data: { + 'error': e.toString(), + }, + ); } } @@ -252,7 +268,31 @@ class TtsController { .any((lang) => lang.startsWith(l2LangCodeShort!))); } - bool isLanguageSupported(LanguageModel lang) => - _availableLangCodes.contains(lang.langCode) || - _availableLangCodes.any((l) => l.startsWith(lang.langCodeShort)); + Future _speakFromChoreo(String text) async { + try { + hasLoadedTextToSpeech = false; + final ttsRes = await chatController?.pangeaController.textToSpeech.get( + TextToSpeechRequest( + text: text, + langCode: l2LangCode ?? LanguageKeys.unknownLanguage, + tokens: [], // TODO: Somehow bring existing tokens to avoid extra choreo token requests + userL1: LanguageKeys.unknownLanguage, + userL2: LanguageKeys.unknownLanguage, + ), + ); + hasLoadedTextToSpeech = true; + if (ttsRes != null) { + final audioContent = base64Decode(ttsRes.audioContent); + await playAudio(audioContent, ttsRes.mimeType); + } + } catch (e, s) { + ErrorHandler.logError( + e: e, + s: s, + data: { + 'text': text, + }, + ); + } + } } diff --git a/lib/pangea/toolbar/widgets/practice_activity/word_text_with_audio_button.dart b/lib/pangea/toolbar/widgets/practice_activity/word_text_with_audio_button.dart index ac6d00dc2..a20eb87c1 100644 --- a/lib/pangea/toolbar/widgets/practice_activity/word_text_with_audio_button.dart +++ b/lib/pangea/toolbar/widgets/practice_activity/word_text_with_audio_button.dart @@ -22,6 +22,26 @@ class WordTextWithAudioButton extends StatefulWidget { class WordAudioButtonState extends State { bool _isPlaying = false; + // initialize as null because we don't know if we need to load + // audio from choreo yet. This shall remain null if user device support + // text to speech + final bool? _isLoadingAudio = null; + + @override + void initState() { + super.initState(); + widget.ttsController.addListener(_onTtsControllerChange); + } + + @override + void dispose() { + widget.ttsController.removeListener(_onTtsControllerChange); + super.dispose(); + } + + void _onTtsControllerChange() { + setState(() {}); + } double get textSize => widget.textSize ?? Theme.of(context).textTheme.titleLarge?.fontSize ?? 16; @@ -41,6 +61,9 @@ class WordAudioButtonState extends State { onExit: (event) => setState(() {}), child: GestureDetector( onTap: () async { + if (_isLoadingAudio == true) { + return; + } if (_isPlaying) { await widget.ttsController.stop(); if (mounted) { @@ -95,10 +118,24 @@ class WordAudioButtonState extends State { ), ), const SizedBox(width: 4), - Icon( - _isPlaying ? Icons.play_arrow : Icons.play_arrow_outlined, - size: textSize, - ), + if (widget.ttsController.hasLoadedTextToSpeech == false) + const Padding( + padding: EdgeInsets.only( + left: 4, + ), // Adds 20 pixels of left padding + child: SizedBox( + width: 16, + height: 16, + child: CircularProgressIndicator( + strokeWidth: 3, + ), + ), + ) + else + Icon( + _isPlaying ? Icons.play_arrow : Icons.play_arrow_outlined, + size: textSize, + ), ], ), ),