ember/fluffychat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: text to speech for all (#2090)

Browse source 
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: ggurdin <ggurdin@gmail.com>
This commit is contained in:
Wilson 2025-03-10 10:14:11 -04:00 • committed by GitHub
parent f59cd0449a
commit 74b0cfd584
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 111 additions and 99 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
lib/pangea/learning_settings/pages/settings_learning.dart
View file

@ -193,7 +193,6 @@ class SettingsLearningController extends State<SettingsLearning> {
case ToolSetting.enableTTS:
return _profile.userSettings.targetLanguage != null &&
_targetLanguage != null &&
tts.isLanguageSupported(_targetLanguage!) &&
toolSettings.enableTTS;
case ToolSetting.enableAutocorrect:
return toolSettings.enableAutocorrect;
@ -201,9 +200,7 @@ class SettingsLearningController extends State<SettingsLearning> {
}
bool get isTTSSupported =>
_profile.userSettings.targetLanguage != null &&
_targetLanguage != null &&
tts.isLanguageSupported(_targetLanguage!);
_profile.userSettings.targetLanguage != null && _targetLanguage != null;
LanguageModel? get selectedSourceLanguage {
return userL1 ?? pangeaController.languageController.systemLanguage;

62
lib/pangea/learning_settings/pages/settings_learning_view.dart
View file

@ -1,7 +1,6 @@
import 'dart:io';
import 'package:flutter/foundation.dart';
import 'package:flutter/gestures.dart';
import 'package:flutter/material.dart';
import 'package:app_settings/app_settings.dart';
@ -18,7 +17,6 @@ import 'package:fluffychat/pangea/learning_settings/widgets/country_picker_tile.
import 'package:fluffychat/pangea/learning_settings/widgets/p_language_dropdown.dart';
import 'package:fluffychat/pangea/learning_settings/widgets/p_settings_switch_list_tile.dart';
import 'package:fluffychat/pangea/spaces/models/space_model.dart';
import 'package:fluffychat/utils/platform_infos.dart';
import 'package:fluffychat/widgets/matrix.dart';
class SettingsLearningView extends StatelessWidget {
@ -242,67 +240,7 @@ class SettingsLearningView extends StatelessWidget {
toolSetting,
value,
),
enabled:
toolSetting == ToolSetting.enableTTS
? controller.isTTSSupported
: true,
),
if (toolSetting == ToolSetting.enableTTS &&
!controller.isTTSSupported)
Row(
children: [
Padding(
padding: const EdgeInsets.only(
right: 16.0,
),
child: Icon(
Icons.info_outlined,
color: Theme.of(context)
.disabledColor,
),
),
Flexible(
child: RichText(
text: TextSpan(
text: L10n.of(context)
.couldNotFindTTS,
style: TextStyle(
color: Theme.of(context)
.disabledColor,
),
children: [
if (PlatformInfos.isWindows ||
PlatformInfos.isAndroid)
TextSpan(
text: L10n.of(context)
.ttsInstructionsHyperlink,
style: const TextStyle(
color: Colors.blue,
fontWeight:
FontWeight.bold,
decoration:
TextDecoration
.underline,
),
recognizer:
TapGestureRecognizer()
..onTap = () {
launchUrlString(
PlatformInfos
.isWindows
? AppConfig
.windowsTTSDownloadInstructions
: AppConfig
.androidTTSDownloadInstructions,
);
},
),
],
),
),
),
],
),
],
),
SwitchListTile.adaptive(

2
lib/pangea/toolbar/controllers/text_to_speech_controller.dart
View file

@ -7,9 +7,9 @@ import 'package:http/http.dart';
import 'package:fluffychat/pangea/common/config/environment.dart';
import 'package:fluffychat/pangea/common/constants/model_keys.dart';
import 'package:fluffychat/pangea/common/controllers/pangea_controller.dart';
import 'package:fluffychat/pangea/common/network/requests.dart';
import 'package:fluffychat/pangea/common/network/urls.dart';
import 'package:fluffychat/pangea/events/models/pangea_token_text_model.dart';
import '../../common/network/requests.dart';
class PangeaAudioEventData {
final String text;

96
lib/pangea/toolbar/controllers/tts_controller.dart
View file

@ -1,4 +1,5 @@
import 'dart:async';
import 'dart:convert';
import 'dart:developer';
import 'package:flutter/foundation.dart';
@ -6,20 +7,22 @@ import 'package:flutter/material.dart';
import 'package:collection/collection.dart';
import 'package:flutter_tts/flutter_tts.dart' as flutter_tts;
import 'package:just_audio/just_audio.dart';
import 'package:matrix/matrix_api_lite/utils/logs.dart';
import 'package:text_to_speech/text_to_speech.dart';
import 'package:fluffychat/pages/chat/chat.dart';
import 'package:fluffychat/pages/chat/events/audio_player.dart';
import 'package:fluffychat/pangea/common/utils/error_handler.dart';
import 'package:fluffychat/pangea/instructions/instructions_enum.dart';
import 'package:fluffychat/pangea/instructions/instructions_show_popup.dart';
import 'package:fluffychat/pangea/learning_settings/models/language_model.dart';
import 'package:fluffychat/pangea/toolbar/widgets/missing_voice_button.dart';
import 'package:fluffychat/pangea/learning_settings/constants/language_constants.dart';
import 'package:fluffychat/pangea/toolbar/controllers/text_to_speech_controller.dart';
import 'package:fluffychat/pangea/user/controllers/user_controller.dart';
import 'package:fluffychat/utils/platform_infos.dart';
import 'package:fluffychat/widgets/matrix.dart';
class TtsController {
class TtsController extends ChangeNotifier {
final ChatController? chatController;
String? get l2LangCode =>
@ -46,9 +49,20 @@ class TtsController {
return PlatformInfos.isWindows;
}
bool _hasLoadedTextToSpeech = false;
bool get hasLoadedTextToSpeech => _hasLoadedTextToSpeech;
set hasLoadedTextToSpeech(bool value) {
if (_hasLoadedTextToSpeech != value) {
_hasLoadedTextToSpeech = value;
notifyListeners();
}
}
@override
Future<void> dispose() async {
await _tts.stop();
await _languageSubscription?.cancel();
super.dispose();
}
void _onError(dynamic message) {
@ -149,22 +163,6 @@ class TtsController {
}
}
Future<void> _showMissingVoicePopup(
BuildContext context,
String targetID,
) async =>
instructionsShowPopup(
context,
InstructionsEnum.missingVoice,
targetID,
showToggle: false,
customContent: const Padding(
padding: EdgeInsets.only(top: 12),
child: MissingVoiceButton(),
),
forceShow: true,
);
Future<void> _showTTSDisabledPopup(
BuildContext context,
String targetID,
@ -189,12 +187,13 @@ class TtsController {
await _setLanguage();
final enableTTS = MatrixState
.pangeaController.userController.profile.toolSettings.enableTTS;
if (_isL2FullySupported && enableTTS) {
await _speak(text);
} else if (!_isL2FullySupported && targetID != null) {
await _showMissingVoicePopup(context, targetID);
} else if (!enableTTS && targetID != null) {
if (enableTTS) {
if (_isL2FullySupported) {
await _speak(text);
} else {
await _speakFromChoreo(text);
}
} else if (targetID != null) {
await _showTTSDisabledPopup(context, targetID);
}
}
@ -242,6 +241,23 @@ class TtsController {
'text': text,
},
);
await _speakFromChoreo(text);
}
}
Future<void> playAudio(Uint8List audioContent, String mimeType) async {
final audioPlayer = AudioPlayer();
try {
await audioPlayer
.setAudioSource(BytesAudioSource(audioContent, mimeType));
await audioPlayer.play();
} catch (e) {
ErrorHandler.logError(
e: 'Error playing audio',
data: {
'error': e.toString(),
},
);
}
}
@ -252,7 +268,31 @@ class TtsController {
.any((lang) => lang.startsWith(l2LangCodeShort!)));
}
bool isLanguageSupported(LanguageModel lang) =>
_availableLangCodes.contains(lang.langCode) ||
_availableLangCodes.any((l) => l.startsWith(lang.langCodeShort));
Future<void> _speakFromChoreo(String text) async {
try {
hasLoadedTextToSpeech = false;
final ttsRes = await chatController?.pangeaController.textToSpeech.get(
TextToSpeechRequest(
text: text,
langCode: l2LangCode ?? LanguageKeys.unknownLanguage,
tokens: [], // TODO: Somehow bring existing tokens to avoid extra choreo token requests
userL1: LanguageKeys.unknownLanguage,
userL2: LanguageKeys.unknownLanguage,
),
);
hasLoadedTextToSpeech = true;
if (ttsRes != null) {
final audioContent = base64Decode(ttsRes.audioContent);
await playAudio(audioContent, ttsRes.mimeType);
}
} catch (e, s) {
ErrorHandler.logError(
e: e,
s: s,
data: {
'text': text,
},
);
}
}
}

45
lib/pangea/toolbar/widgets/practice_activity/word_text_with_audio_button.dart
View file

@ -22,6 +22,26 @@ class WordTextWithAudioButton extends StatefulWidget {
class WordAudioButtonState extends State<WordTextWithAudioButton> {
bool _isPlaying = false;
// initialize as null because we don't know if we need to load
// audio from choreo yet. This shall remain null if user device support
// text to speech
final bool? _isLoadingAudio = null;
@override
void initState() {
super.initState();
widget.ttsController.addListener(_onTtsControllerChange);
}
@override
void dispose() {
widget.ttsController.removeListener(_onTtsControllerChange);
super.dispose();
}
void _onTtsControllerChange() {
setState(() {});
}
double get textSize =>
widget.textSize ?? Theme.of(context).textTheme.titleLarge?.fontSize ?? 16;
@ -41,6 +61,9 @@ class WordAudioButtonState extends State<WordTextWithAudioButton> {
onExit: (event) => setState(() {}),
child: GestureDetector(
onTap: () async {
if (_isLoadingAudio == true) {
return;
}
if (_isPlaying) {
await widget.ttsController.stop();
if (mounted) {
@ -95,10 +118,24 @@ class WordAudioButtonState extends State<WordTextWithAudioButton> {
),
),
const SizedBox(width: 4),
Icon(
_isPlaying ? Icons.play_arrow : Icons.play_arrow_outlined,
size: textSize,
),
if (widget.ttsController.hasLoadedTextToSpeech == false)
const Padding(
padding: EdgeInsets.only(
left: 4,
), // Adds 20 pixels of left padding
child: SizedBox(
width: 16,
height: 16,
child: CircularProgressIndicator(
strokeWidth: 3,
),
),
)
else
Icon(
_isPlaying ? Icons.play_arrow : Icons.play_arrow_outlined,
size: textSize,
),
],
),
),