ember/fluffychat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

finished saving text as transcript event, going to change to rep event though

Browse source
This commit is contained in:
William Jordan-Cooley 2024-04-13 12:11:24 -04:00
parent b1c26f0572
commit f3bb717245
14 changed files with 737 additions and 688 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
assets/l10n/intl_en.arb
View file

@ -2415,9 +2415,6 @@
"seconds": {}
}
},
"pleaseEnterANumber": "Please enter a number greater than 0",
"archiveRoomDescription": "The chat will be moved to the archive. Other users will be able to see that you have left the chat.",
"roomUpgradeDescription": "The chat will then be recreated with the new room version. All participants will be notified that they need to switch to the new chat. You can find out more about room versions at https://spec.matrix.org/latest/rooms/",
"allCorrect": "That's how I would say it! Nice!",
"newWayAllGood": "That's not how I would have said it but it looks good!",
"othersAreBetter": "Hm, there might be a better way to say that.",
@ -2568,15 +2565,7 @@
"placeholders": {}
},
"copyClassLink": "Copy invite link",
"@copyClassLink": {
"type": "text",
"placeholders": {}
},
"copyClassLinkDesc": "Clicking this link will take students to the app, direct them to make an account and they will automatically join this space.",
"@copyClassLink": {
"type": "text",
"placeholders": {}
},
"copyClassCode": "Copy invite code",
"inviteStudentByUserName": "Invite student by username",
"@inviteStudentByUserName": {
@ -2748,11 +2737,6 @@
"type": "text",
"placeholders": {}
},
"errorPleaseRefresh": "We're looking into it! Please reload and try again.",
"@errorPleaseRefresh": {
"type": "text",
"placeholders": {}
},
"joinWithClassCode": "Join class or exchange",
"@joinWithClassCode": {
"type": "text",
@ -2973,26 +2957,6 @@
"type": "text",
"placeholders": {}
},
"error502504Title": "Wow, there are a lot of students online!",
"@error502504Title": {
"type": "text",
"placeholders": {}
},
"error502504Desc": "Translation and grammar tools may be slow or unavailable while the Pangea bots catch up.",
"@error502504Desc": {
"type": "text",
"placeholders": {}
},
"error404Title": "Translation error!",
"@error404Title": {
"type": "text",
"placeholders": {}
},
"error404Desc": "Pangea Bot isn't sure how to translate that...",
"@error404Desc": {
"type": "text",
"placeholders": {}
},
"errorDisableIT": "Translation assistance is turned off.",
"errorDisableIGC": "Grammar assistance is turned off.",
"errorDisableLanguageAssistance": "Translation assistance and grammar assistance are turned off.",
@ -3095,11 +3059,6 @@
"type": "text",
"placeholders": {}
},
"classDescription": "Space Description",
"@classDescription": {
"type": "text",
"placeholders": {}
},
"inviteStudentByUserNameDesc": "If your student already has an account, you can search for them.",
"@inviteStudentByUserNameDesc": {
"type": "text",
@ -3116,7 +3075,6 @@
"clickMessageTitle": "Need help?",
"clickMessageBody": "Click messages to access definitions, translations, and audio!",
"understandingMessagesTitle": "Definitions and translations!",
"addToClass": "Add this chat to ",
"understandingMessagesBody": "Click underlined words for definitions. Translate with message options (upper right).",
"allDone": "All done!",
"vocab": "Vocabulary",
@ -3649,7 +3607,6 @@
"user": {}
}
},
"decline": "Decline",
"declinedInvitation": "Declined invitation",
"acceptedInvitation": "Accepted invitation",
"youreInvited": "📩 You're invited!",
@ -3728,7 +3685,6 @@
},
"acceptSelection": "Accept Correction",
"acceptSelectionAnyway": "Use this anyway",
"replace": "Make correction",
"makingActivity": "Making activity",
"why": "Why?",
"definition": "Definition",
@ -3751,12 +3707,6 @@
}
},
"noTeachersFound": "No teachers found to report to",
"pushNotificationsNotAvailable": "Push notifications not available",
"learnMore": "Learn more",
"banUserDescription": "The user will be banned from the chat and will not be able to enter the chat again until they are unbanned.",
"unbanUserDescription": "The user will be able to enter the chat again if they try.",
"kickUserDescription": "The user is kicked out of the chat but not banned. In public chats, the user can rejoin at any time.",
"makeAdminDescription": "Once you make this user admin, you may not be able to undo this as they will then have the same permissions as you.",
"pleaseEnterANumber": "Please enter a number greater than 0",
"archiveRoomDescription": "The chat will be moved to the archive. Other users will be able to see that you have left the chat.",
"roomUpgradeDescription": "The chat will then be recreated with the new room version. All participants will be notified that they need to switch to the new chat. You can find out more about room versions at https://spec.matrix.org/latest/rooms/",
@ -3776,10 +3726,6 @@
}
},
"searchChatsRooms": "Search for #chats, @users...",
"groupName": "Group name",
"createGroupAndInviteUsers": "Create a group and invite users",
"groupCanBeFoundViaSearch": "Group can be found via search",
"inNoSpaces": "You are not a member of any classes or exchanges",
"createClass": "Create class",
"createExchange": "Create exchange",
"viewArchive": "View Archive",
@ -3886,7 +3832,7 @@
"enableModerationDesc": "Enable automatic moderation to review messages before they are sent",
"conversationLanguageLevel": "What is the language level of this conversation?",
"showDefinition": "Show Definition",
"acceptedKeyVerification": "{sender} accepted key verification",
"acceptedKeyVerification": "{sender} accepted key verification",
"@acceptedKeyVerification": {
"type": "text",
"placeholders": {
@ -3941,6 +3887,7 @@
"more": "More",
"translationTooltip": "Translate",
"audioTooltip": "Play Audio",
"transcriptTooltip": "Transcript",
"certifyAge": "I certify that I am over {age} years of age",
"@certifyAge": {
"type": "text",

3
assets/l10n/intl_es.arb
View file

@ -4572,6 +4572,7 @@
"more": "Más",
"translationTooltip": "Traducir",
"audioTooltip": "Reproducir audio",
"transcriptTooltip": "Transcripción",
"yourBirthdayPleaseShort": "Seleccione su grupo de edad",
"certifyAge": "Certifico que soy mayor de {age} años",
"@certifyAge": {
@ -4587,4 +4588,4 @@
"joinToView": "Únete a esta sala para ver los detalles",
"autoPlayTitle": "Reproducción automática de mensajes",
"autoPlayDesc": "Cuando está activado, el audio de texto a voz de los mensajes se reproducirá automáticamente cuando se seleccione."
}
}

2
lib/pages/chat/events/message_content.dart
View file

@ -341,7 +341,7 @@ class MessageContent extends StatelessWidget {
),
onListen: () => toolbarController?.showToolbar(
context,
mode: MessageMode.conversion,
mode: MessageMode.textToSpeech,
),
),
enableInteractiveSelection:

4
lib/pangea/constants/pangea_event_types.dart
View file

@ -2,6 +2,8 @@ class PangeaEventTypes {
static const classSettings = "pangea.class";
static const pangeaExchange = "p.exchange";
static const transcript = "pangea.transcript";
static const rules = "p.rules";
static const studentAnalyticsSummary = "pangea.usranalytics";
@ -18,4 +20,6 @@ class PangeaEventTypes {
static const botOptions = "pangea.bot_options";
static const userAge = "pangea.user_age";
static const String report = 'm.report';
}

3
lib/pangea/constants/pangea_message_types.dart
View file

@ -1,3 +0,0 @@
class PangeaMessageTypes {
static String report = 'm.report';
}

43
lib/pangea/controllers/speech_to_text_controller.dart
View file

@ -1,8 +1,11 @@
import 'dart:async';
import 'dart:convert';
import 'package:fluffychat/pangea/constants/pangea_event_types.dart';
import 'package:fluffychat/pangea/controllers/pangea_controller.dart';
import 'package:fluffychat/pangea/extensions/pangea_room_extension.dart';
import 'package:fluffychat/pangea/models/speech_to_text_models.dart';
import 'package:fluffychat/pangea/utils/error_handler.dart';
import 'package:flutter/foundation.dart';
import 'package:http/http.dart';
@ -28,7 +31,7 @@ class SpeechToTextController {
}
void _initializeCacheClearing() {
const duration = Duration(minutes: 15);
const duration = Duration(minutes: 2);
_cacheClearTimer = Timer.periodic(duration, (Timer t) => _clearCache());
}
@ -41,7 +44,8 @@ class SpeechToTextController {
}
Future<SpeechToTextResponseModel> get(
SpeechToTextRequestModel requestModel) async {
SpeechToTextRequestModel requestModel,
) async {
final int cacheKey = requestModel.hashCode;
if (_cache.containsKey(cacheKey)) {
@ -52,11 +56,35 @@ class SpeechToTextController {
requestModel: requestModel,
);
_cache[cacheKey] = _SpeechToTextCacheItem(data: response);
return response;
}
}
static Future<SpeechToTextResponseModel> _fetchResponse({
Future<void> saveTranscriptAsMatrixEvent(
SpeechToTextResponseModel response,
SpeechToTextRequestModel requestModel,
) {
if (requestModel.audioEvent == null) {
debugPrint(
'Audio event is null, case of giving speech to text before message sent, currently not implemented',
);
return Future.value(null);
}
debugPrint('Saving transcript as matrix event');
final json = response.toJson();
requestModel.audioEvent?.room.sendPangeaEvent(
content: response.toJson(),
parentEventId: requestModel.audioEvent!.eventId,
type: PangeaEventTypes.transcript,
);
debugPrint('Transcript saved as matrix event');
return Future.value(null);
}
Future<SpeechToTextResponseModel> _fetchResponse({
required String accessToken,
required SpeechToTextRequestModel requestModel,
}) async {
@ -72,7 +100,14 @@ class SpeechToTextController {
if (res.statusCode == 200) {
final Map<String, dynamic> json = jsonDecode(utf8.decode(res.bodyBytes));
return SpeechToTextResponseModel.fromJson(json);
final response = SpeechToTextResponseModel.fromJson(json);
saveTranscriptAsMatrixEvent(response, requestModel).onError(
(error, stackTrace) => ErrorHandler.logError(e: error, s: stackTrace),
);
return response;
} else {
debugPrint('Error converting speech to text: ${res.body}');
throw Exception('Failed to convert speech to text');

16
lib/pangea/enum/message_mode_enum.dart
View file

@ -1,15 +1,17 @@
import 'package:flutter/material.dart';
import 'package:flutter_gen/gen_l10n/l10n.dart';
enum MessageMode { conversion, translation, definition }
enum MessageMode { translation, definition, speechToText, textToSpeech }
extension MessageModeExtension on MessageMode {
IconData icon(bool isAudioMessage) {
IconData get icon {
switch (this) {
case MessageMode.translation:
return Icons.g_translate;
case MessageMode.conversion:
case MessageMode.textToSpeech:
return Icons.play_arrow;
case MessageMode.speechToText:
return Icons.mic;
//TODO change icon for audio messages
case MessageMode.definition:
return Icons.book;
@ -22,8 +24,10 @@ extension MessageModeExtension on MessageMode {
switch (this) {
case MessageMode.translation:
return L10n.of(context)!.translations;
case MessageMode.conversion:
case MessageMode.textToSpeech:
return L10n.of(context)!.messageAudio;
case MessageMode.speechToText:
return L10n.of(context)!.transcriptTooltip;
case MessageMode.definition:
return L10n.of(context)!.definitions;
default:
@ -36,8 +40,10 @@ extension MessageModeExtension on MessageMode {
switch (this) {
case MessageMode.translation:
return L10n.of(context)!.translationTooltip;
case MessageMode.conversion:
case MessageMode.textToSpeech:
return L10n.of(context)!.audioTooltip;
case MessageMode.speechToText:
return L10n.of(context)!.transcriptTooltip;
case MessageMode.definition:
return L10n.of(context)!.define;
default:

81
lib/pangea/models/pangea_message_event.dart
View file

@ -2,14 +2,15 @@ import 'dart:convert';
import 'package:collection/collection.dart';
import 'package:fluffychat/pangea/constants/model_keys.dart';
import 'package:fluffychat/pangea/constants/pangea_message_types.dart';
import 'package:fluffychat/pangea/controllers/text_to_speech_controller.dart';
import 'package:fluffychat/pangea/enum/audio_encoding_enum.dart';
import 'package:fluffychat/pangea/extensions/pangea_room_extension.dart';
import 'package:fluffychat/pangea/models/choreo_record.dart';
import 'package:fluffychat/pangea/models/class_model.dart';
import 'package:fluffychat/pangea/models/message_data_models.dart';
import 'package:fluffychat/pangea/models/pangea_match_model.dart';
import 'package:fluffychat/pangea/models/pangea_representation_event.dart';
import 'package:fluffychat/pangea/models/speech_to_text_models.dart';
import 'package:fluffychat/pangea/utils/bot_name.dart';
import 'package:fluffychat/pangea/widgets/chat/message_audio_card.dart';
import 'package:flutter/material.dart';
@ -56,6 +57,8 @@ class PangeaMessageEvent {
Room get room => _event.room;
bool get isAudioMessage => _event.messageType == MessageTypes.Audio;
Event? _latestEditCache;
Event get _latestEdit => _latestEditCache ??= _event
.aggregatedEvents(
@ -267,6 +270,78 @@ class PangeaMessageEvent {
null;
}).toSet();
Set<Event> get transcriptionEvents => _event.aggregatedEvents(
timeline,
PangeaEventTypes.transcript,
);
Event? get transcriptionEvent => transcriptionEvents.lastOrNull;
Future<SpeechToTextResponseModel?> getSpeechToTextLocal() async {
if (transcriptionEvent == null) return null;
return SpeechToTextResponseModel.fromJson(
transcriptionEvent!.content[PangeaEventTypes.transcript]
as Map<String, dynamic>,
);
}
Future<SpeechToTextResponseModel?> getSpeechToTextGlobal(
String l1Code,
String l2Code,
) async {
if (!isAudioMessage) {
ErrorHandler.logError(
e: 'Message is not an audio message ${_event.eventId}',
s: StackTrace.current,
data: _event.content,
);
return null;
}
if (transcriptionEvent != null) return getSpeechToTextLocal();
final matrixFile = await _event.downloadAndDecryptAttachment();
// Pangea#
// File? file;
// TODO: Test on mobile and see if we need this case, doeesn't seem so
// if (!kIsWeb) {
// final tempDir = await getTemporaryDirectory();
// final fileName = Uri.encodeComponent(
// // #Pangea
// // widget.event.attachmentOrThumbnailMxcUrl()!.pathSegments.last,
// widget.messageEvent.event
// .attachmentOrThumbnailMxcUrl()!
// .pathSegments
// .last,
// // Pangea#
// );
// file = File('${tempDir.path}/${fileName}_${matrixFile.name}');
// await file.writeAsBytes(matrixFile.bytes);
// }
// audioFile = file;
final SpeechToTextResponseModel response =
await MatrixState.pangeaController.speechToText.get(
SpeechToTextRequestModel(
audioContent: matrixFile.bytes,
audioEvent: _event,
config: SpeechToTextAudioConfigModel(
encoding: mimeTypeToAudioEncoding(matrixFile.mimeType),
//this is the default in the RecordConfig in record package
//TODO: check if this is the correct value and make it a constant somewhere
sampleRateHertz: 44100,
userL1: l1Code,
userL2: l2Code,
),
),
);
return response;
}
List<RepresentationEvent>? _representations;
List<RepresentationEvent> get representations {
if (_representations != null) return _representations!;
@ -431,6 +506,8 @@ class PangeaMessageEvent {
),
);
},
).onError(
(error, stackTrace) => ErrorHandler.logError(e: error, s: stackTrace),
);
return pangeaRep;
@ -456,7 +533,7 @@ class PangeaMessageEvent {
_event.room.isSpaceAdmin &&
_event.senderId != BotName.byEnvironment &&
!room.isUserSpaceAdmin(_event.senderId) &&
_event.messageType != PangeaMessageTypes.report;
_event.messageType != PangeaEventTypes.report;
String get messageDisplayLangCode {
final bool immersionMode = MatrixState

30
lib/pangea/models/speech_to_text_models.dart
View file

@ -2,6 +2,7 @@ import 'dart:convert';
import 'package:fluffychat/pangea/enum/audio_encoding_enum.dart';
import 'package:flutter/foundation.dart';
import 'package:matrix/matrix.dart';
class SpeechToTextAudioConfigModel {
final AudioEncodingEnum encoding;
@ -33,10 +34,12 @@ class SpeechToTextAudioConfigModel {
class SpeechToTextRequestModel {
final Uint8List audioContent;
final SpeechToTextAudioConfigModel config;
final Event? audioEvent;
SpeechToTextRequestModel({
required this.audioContent,
required this.config,
this.audioEvent,
});
Map<String, dynamic> toJson() => {
@ -68,7 +71,7 @@ class WordInfo {
final String word;
final Duration? startTime;
final Duration? endTime;
final double? confidence;
final int? confidence;
WordInfo({
required this.word,
@ -85,13 +88,20 @@ class WordInfo {
endTime: json['end_time'] != null
? Duration(milliseconds: json['end_time'])
: null,
confidence: (json['confidence'] as num?)?.toDouble(),
confidence: json['confidence'],
);
Map<String, dynamic> toJson() => {
"word": word,
"start_time": startTime?.inMilliseconds,
"end_time": endTime?.inMilliseconds,
"confidence": confidence,
};
}
class Transcript {
final String transcript;
final double confidence;
final int confidence;
final List<WordInfo> words;
Transcript({
@ -106,6 +116,12 @@ class Transcript {
words:
(json['words'] as List).map((e) => WordInfo.fromJson(e)).toList(),
);
Map<String, dynamic> toJson() => {
"transcript": transcript,
"confidence": confidence,
"words": words.map((e) => e.toJson()).toList(),
};
}
class SpeechToTextResult {
@ -119,6 +135,10 @@ class SpeechToTextResult {
.map((e) => Transcript.fromJson(e))
.toList(),
);
Map<String, dynamic> toJson() => {
"transcripts": transcripts.map((e) => e.toJson()).toList(),
};
}
class SpeechToTextResponseModel {
@ -134,4 +154,8 @@ class SpeechToTextResponseModel {
.map((e) => SpeechToTextResult.fromJson(e))
.toList(),
);
Map<String, dynamic> toJson() => {
"results": results.map((e) => e.toJson()).toList(),
};
}

12
lib/pangea/utils/report_message.dart
View file

@ -1,12 +1,10 @@
import 'package:flutter/material.dart';
import 'package:flutter_gen/gen_l10n/l10n.dart';
import 'package:matrix/matrix.dart';
import 'package:fluffychat/pangea/constants/pangea_message_types.dart';
import 'package:fluffychat/pangea/constants/pangea_event_types.dart';
import 'package:fluffychat/pangea/extensions/client_extension.dart';
import 'package:fluffychat/pangea/extensions/pangea_room_extension.dart';
import 'package:fluffychat/widgets/matrix.dart';
import 'package:flutter/material.dart';
import 'package:flutter_gen/gen_l10n/l10n.dart';
import 'package:matrix/matrix.dart';
Future<void> reportMessage(
BuildContext context,
@ -66,7 +64,7 @@ Future<void> reportMessage(
final String message = "$messageTitle\n\n$messageBody";
for (final Room reportDM in reportDMs) {
final event = <String, dynamic>{
'msgtype': PangeaMessageTypes.report,
'msgtype': PangeaEventTypes.report,
'body': message,
};
await reportDM.sendEvent(event);

159
lib/pangea/widgets/chat/message_speech_to_text_card.dart
View file

@ -1,16 +1,10 @@
import 'dart:developer';
import 'package:fluffychat/pangea/enum/audio_encoding_enum.dart';
import 'package:fluffychat/pangea/models/message_data_models.dart';
import 'package:fluffychat/pangea/models/pangea_message_event.dart';
import 'package:fluffychat/pangea/models/speech_to_text_models.dart';
import 'package:fluffychat/pangea/utils/bot_style.dart';
import 'package:fluffychat/pangea/utils/error_handler.dart';
import 'package:fluffychat/utils/localized_exception_extension.dart';
import 'package:fluffychat/pangea/widgets/igc/card_error_widget.dart';
import 'package:fluffychat/widgets/matrix.dart';
import 'package:flutter/foundation.dart';
import 'package:flutter/material.dart';
import 'package:matrix/matrix.dart';
class MessageSpeechToTextCard extends StatefulWidget {
final PangeaMessageEvent messageEvent;
@ -24,15 +18,10 @@ class MessageSpeechToTextCard extends StatefulWidget {
MessageSpeechToTextCardState createState() => MessageSpeechToTextCardState();
}
enum AudioFileStatus { notDownloaded, downloading, downloaded }
class MessageSpeechToTextCardState extends State<MessageSpeechToTextCard> {
PangeaRepresentation? repEvent;
String? transcription;
SpeechToTextResponseModel? speechToTextResponse;
bool _fetchingTranscription = true;
AudioFileStatus status = AudioFileStatus.notDownloaded;
MatrixFile? matrixFile;
// File? audioFile;
Object? error;
String? get l1Code =>
MatrixState.pangeaController.languageController.activeL1Code(
@ -43,124 +32,41 @@ class MessageSpeechToTextCardState extends State<MessageSpeechToTextCard> {
roomID: widget.messageEvent.room.id,
);
// get transcription from local events
Future<String> getLocalTranscription() async {
return "This is a dummy transcription";
}
// This code is duplicated from audio_player.dart. Is there some way to reuse that code?
Future<void> _downloadAction() async {
// #Pangea
// if (status != AudioFileStatus.notDownloaded) return;
if (status != AudioFileStatus.notDownloaded) {
return;
}
// Pangea#
setState(() => status = AudioFileStatus.downloading);
try {
// #Pangea
// final matrixFile = await widget.event.downloadAndDecryptAttachment();
final matrixFile =
await widget.messageEvent.event.downloadAndDecryptAttachment();
// Pangea#
// File? file;
// TODO: Test on mobile and see if we need this case
// if (!kIsWeb) {
// final tempDir = await getTemporaryDirectory();
// final fileName = Uri.encodeComponent(
// // #Pangea
// // widget.event.attachmentOrThumbnailMxcUrl()!.pathSegments.last,
// widget.messageEvent.event
// .attachmentOrThumbnailMxcUrl()!
// .pathSegments
// .last,
// // Pangea#
// );
// file = File('${tempDir.path}/${fileName}_${matrixFile.name}');
// await file.writeAsBytes(matrixFile.bytes);
// }
// audioFile = file;
this.matrixFile = matrixFile;
status = AudioFileStatus.downloaded;
} catch (e, s) {
Logs().v('Could not download audio file', e, s);
ScaffoldMessenger.of(context).showSnackBar(
SnackBar(
content: Text(e.toLocalizedString(context)),
),
);
}
}
AudioEncodingEnum? get encoding {
if (matrixFile == null) return null;
return mimeTypeToAudioEncoding(matrixFile!.mimeType);
}
// call API to transcribe audio
Future<String?> transcribeAudio() async {
await _downloadAction();
final localmatrixFile = matrixFile;
final info = matrixFile?.info;
if (matrixFile == null) {
debugger(when: kDebugMode);
ErrorHandler.logError(
e: 'Audio file or matrix file is null ${widget.messageEvent.eventId}',
s: StackTrace.current,
data: widget.messageEvent.event.content,
);
return null;
}
if (l1Code == null || l2Code == null) {
debugger(when: kDebugMode);
ErrorHandler.logError(
e: 'Language codes are null ${widget.messageEvent.eventId}',
s: StackTrace.current,
data: widget.messageEvent.event.content,
);
return null;
}
final SpeechToTextResponseModel response =
await MatrixState.pangeaController.speechToText.get(
SpeechToTextRequestModel(
audioContent: matrixFile!.bytes,
config: SpeechToTextAudioConfigModel(
encoding: encoding ?? AudioEncodingEnum.encodingUnspecified,
//this is the default in the RecordConfig in record package
sampleRateHertz: 44100,
userL1: l1Code!,
userL2: l2Code!,
),
),
);
return response.results.first.transcripts.first.transcript;
}
String? get transcription => speechToTextResponse
?.results.firstOrNull?.transcripts.firstOrNull?.transcript;
// look for transcription in message event
// if not found, call API to transcribe audio
Future<void> loadTranscription() async {
// transcription ??= await getLocalTranscription();
transcription ??= await transcribeAudio();
setState(() => _fetchingTranscription = false);
Future<void> getSpeechToText() async {
try {
if (l1Code == null || l2Code == null) {
throw Exception('Language selection not found');
}
speechToTextResponse ??=
await widget.messageEvent.getSpeechToTextGlobal(l1Code!, l2Code!);
} catch (e, s) {
error = e;
ErrorHandler.logError(
e: e,
s: s,
data: widget.messageEvent.event.content,
);
} finally {
setState(() => _fetchingTranscription = false);
}
}
@override
void initState() {
super.initState();
loadTranscription();
getSpeechToText();
}
@override
Widget build(BuildContext context) {
// if (!_fetchingTranscription && repEvent == null && transcription == null) {
// return const CardErrorWidget();
// }
if (!_fetchingTranscription && speechToTextResponse == null) {
return CardErrorWidget(error: error);
}
return Padding(
padding: const EdgeInsets.all(8),
@ -173,15 +79,10 @@ class MessageSpeechToTextCardState extends State<MessageSpeechToTextCard> {
color: Theme.of(context).colorScheme.primary,
),
)
: transcription != null
? Text(
transcription!,
style: BotStyle.text(context),
)
: Text(
repEvent!.text,
style: BotStyle.text(context),
),
: Text(
transcription!,
style: BotStyle.text(context),
),
);
}
}

55
lib/pangea/widgets/chat/message_toolbar.dart
View file

@ -176,13 +176,21 @@ class MessageToolbarState extends State<MessageToolbar> {
case MessageMode.translation:
showTranslation();
break;
case MessageMode.conversion:
showConversion();
case MessageMode.textToSpeech:
showTextToSpeech();
break;
case MessageMode.speechToText:
showSpeechToText();
break;
case MessageMode.definition:
showDefinition();
break;
default:
ErrorHandler.logError(
e: "Invalid toolbar mode",
s: StackTrace.current,
data: {"newMode": newMode},
);
break;
}
}
@ -200,21 +208,22 @@ class MessageToolbarState extends State<MessageToolbar> {
);
}
void showConversion() {
debugPrint("show conversion");
if (isAudioMessage) {
debugPrint("is audio message");
toolbarContent = MessageSpeechToTextCard(
messageEvent: widget.pangeaMessageEvent,
);
} else {
toolbarContent = MessageAudioCard(
messageEvent: widget.pangeaMessageEvent,
);
}
void showTextToSpeech() {
debugPrint("show text to speech");
toolbarContent = MessageAudioCard(
messageEvent: widget.pangeaMessageEvent,
);
}
void showSpeechToText() {
debugPrint("show speech to text");
toolbarContent = MessageSpeechToTextCard(
messageEvent: widget.pangeaMessageEvent,
);
}
void showDefinition() {
debugPrint("show definition");
if (widget.textSelection.selectedText == null ||
widget.textSelection.selectedText!.isEmpty) {
toolbarContent = const SelectToDefine();
@ -231,10 +240,6 @@ class MessageToolbarState extends State<MessageToolbar> {
);
}
bool get isAudioMessage {
return widget.pangeaMessageEvent.event.messageType == MessageTypes.Audio;
}
void showImage() {}
void spellCheck() {}
@ -259,7 +264,9 @@ class MessageToolbarState extends State<MessageToolbar> {
) ??
true;
autoplay
? updateMode(MessageMode.conversion)
? updateMode(widget.pangeaMessageEvent.isAudioMessage
? MessageMode.speechToText
: MessageMode.textToSpeech)
: updateMode(MessageMode.translation);
});
@ -322,13 +329,19 @@ class MessageToolbarState extends State<MessageToolbar> {
Row(
mainAxisSize: MainAxisSize.min,
children: MessageMode.values.map((mode) {
if (mode == MessageMode.definition && isAudioMessage) {
if ([MessageMode.definition, MessageMode.textToSpeech]
.contains(mode) &&
widget.pangeaMessageEvent.isAudioMessage) {
return const SizedBox.shrink();
}
if (mode == MessageMode.speechToText &&
!widget.pangeaMessageEvent.isAudioMessage) {
return const SizedBox.shrink();
}
return Tooltip(
message: mode.tooltip(context),
child: IconButton(
icon: Icon(mode.icon(isAudioMessage)),
icon: Icon(mode.icon),
color: currentMode == mode
? Theme.of(context).colorScheme.primary
: null,

5
lib/pangea/widgets/igc/pangea_rich_text.dart
View file

@ -87,7 +87,8 @@ class PangeaRichTextState extends State<PangeaRichText> {
context: context,
langCode: widget.pangeaMessageEvent.messageDisplayLangCode,
)
.onError((error, stackTrace) => ErrorHandler.logError())
.onError((error, stackTrace) =>
ErrorHandler.logError(e: error, s: stackTrace))
.then((event) {
repEvent = event;
widget.toolbarController?.toolbar?.textSelection.setMessageText(
@ -158,7 +159,7 @@ class PangeaRichTextState extends State<PangeaRichText> {
),
onListen: () => widget.toolbarController?.showToolbar(
context,
mode: MessageMode.conversion,
mode: MessageMode.textToSpeech,
),
),
TextSpan(

955
needed-translations.txt
View file

File diff suppressed because it is too large Load diff