ember/fluffychat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore: move STT into message bubble (#2564)

Browse source
This commit is contained in:
ggurdin 2025-04-28 10:22:32 -04:00 • committed by GitHub
parent c0680b5294
commit 51b5ec1163
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 79 additions and 180 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
lib/pangea/toolbar/reading_assistance_input_row/reading_assistance_input_bar.dart
View file

@ -9,7 +9,6 @@ import 'package:fluffychat/pangea/toolbar/enums/message_mode_enum.dart';
import 'package:fluffychat/pangea/toolbar/enums/reading_assistance_mode_enum.dart';
import 'package:fluffychat/pangea/toolbar/widgets/message_mode_locked_card.dart';
import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart';
import 'package:fluffychat/pangea/toolbar/widgets/message_speech_to_text_card.dart';
import 'package:fluffychat/pangea/toolbar/widgets/message_translation_card.dart';
import 'package:fluffychat/pangea/toolbar/widgets/practice_activity/practice_activity_card.dart';
@ -43,14 +42,7 @@ class ReadingAssistanceInputBar extends StatelessWidget {
: null;
if (overlayController.pangeaMessageEvent?.isAudioMessage == true) {
if (!['audio/wav', 'audio/x-wav']
.contains(overlayController.pangeaMessageEvent!.mimetype)) {
return ReactionsPicker(controller);
}
content = MessageSpeechToTextCard(
messageEvent: overlayController.pangeaMessageEvent!,
);
return ReactionsPicker(controller);
} else {
switch (overlayController.toolbarMode) {
case MessageMode.messageSpeechToText:

5
lib/pangea/toolbar/widgets/message_selection_overlay.dart
View file

@ -183,11 +183,6 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
}
Future<void> _setInitialToolbarMode() async {
if (pangeaMessageEvent?.isAudioMessage ?? false) {
updateToolbarMode(MessageMode.listening);
return;
}
// 1) if we have a hidden word activity, then we should start with that
if (practiceSelection?.hasHiddenWordActivity ?? false) {
updateToolbarMode(MessageMode.practiceActivity);

14
lib/pangea/toolbar/widgets/message_selection_positioner.dart
View file

@ -484,19 +484,23 @@ class MessageSelectionPositionerState extends State<MessageSelectionPositioner>
// measurement for items in the toolbar
bool get showPracticeButtons =>
bool get _showButtons =>
(widget.pangeaMessageEvent?.shouldShowToolbar ?? false) &&
widget.pangeaMessageEvent?.event.messageType == MessageTypes.Text &&
(widget.pangeaMessageEvent?.messageDisplayLangIsL2 ?? false) &&
(widget.pangeaMessageEvent?.messageDisplayLangIsL2 ?? false);
bool get showPracticeButtons =>
_showButtons &&
widget.overlayController.readingAssistanceMode ==
ReadingAssistanceMode.practiceMode;
bool get showSelectionButtons =>
widget.overlayController.readingAssistanceMode ==
ReadingAssistanceMode.selectMode;
_showButtons &&
[ReadingAssistanceMode.selectMode, null]
.contains(widget.overlayController.readingAssistanceMode);
double get _selectionButtonsHeight {
return AppConfig.toolbarButtonsHeight;
return showSelectionButtons ? AppConfig.toolbarButtonsHeight : 0;
}
bool get _hasReactions {

204
lib/pangea/toolbar/widgets/message_speech_to_text_card.dart
View file

@ -6,23 +6,19 @@ import 'package:flutter/material.dart';
import 'package:flutter_gen/gen_l10n/l10n.dart';
import 'package:fluffychat/config/app_config.dart';
import 'package:fluffychat/pangea/choreographer/widgets/igc/card_error_widget.dart';
import 'package:fluffychat/pangea/common/utils/error_handler.dart';
import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart';
import 'package:fluffychat/pangea/instructions/instructions_enum.dart';
import 'package:fluffychat/pangea/instructions/instructions_inline_tooltip.dart';
import 'package:fluffychat/pangea/toolbar/models/speech_to_text_models.dart';
import 'package:fluffychat/pangea/toolbar/widgets/icon_number_widget.dart';
import 'package:fluffychat/pangea/toolbar/widgets/toolbar_content_loading_indicator.dart';
import 'package:fluffychat/widgets/matrix.dart';
import '../../bot/utils/bot_style.dart';
class MessageSpeechToTextCard extends StatefulWidget {
final PangeaMessageEvent messageEvent;
final Color textColor;
const MessageSpeechToTextCard({
super.key,
required this.messageEvent,
required this.textColor,
});
@override
@ -30,29 +26,33 @@ class MessageSpeechToTextCard extends StatefulWidget {
}
class MessageSpeechToTextCardState extends State<MessageSpeechToTextCard> {
SpeechToTextModel? speechToTextResponse;
SpeechToTextModel? _speechToTextResponse;
bool _fetchingTranscription = true;
Object? error;
STTToken? selectedToken;
TextSpan? transcriptText;
String? get l1Code =>
MatrixState.pangeaController.languageController.activeL1Code();
String? get l2Code =>
MatrixState.pangeaController.languageController.activeL2Code();
@override
void initState() {
super.initState();
_fetchTranscription();
}
// look for transcription in message event
// if not found, call API to transcribe audio
Future<void> getSpeechToText() async {
Future<void> _fetchTranscription() async {
try {
if (l1Code == null || l2Code == null) {
throw Exception('Language selection not found');
}
speechToTextResponse ??=
await widget.messageEvent.getSpeechToText(l1Code!, l2Code!);
debugPrint(
'Speech to text transcript: ${speechToTextResponse?.transcript.text}',
_speechToTextResponse ??= await widget.messageEvent.getSpeechToText(
l1Code!,
l2Code!,
);
} catch (e, s) {
debugger(when: kDebugMode);
@ -69,153 +69,51 @@ class MessageSpeechToTextCardState extends State<MessageSpeechToTextCard> {
}
}
TextSpan _buildTranscriptText(BuildContext context) {
try {
final Transcript transcript = speechToTextResponse!.transcript;
final List<InlineSpan> spans = [];
String remainingFullText = transcript.text;
if (transcript.sttTokens.isEmpty) {
return TextSpan(
text: remainingFullText,
style: BotStyle.text(context),
);
}
for (final token in transcript.sttTokens) {
final offset = remainingFullText.indexOf(token.token.text.content);
if (offset == -1) continue;
final length = token.length;
if (remainingFullText.substring(0, offset).trim().isNotEmpty) {
remainingFullText = remainingFullText.substring(offset);
continue;
}
if (offset > 0) {
// Add any plain text before the token
spans.add(
TextSpan(text: remainingFullText.substring(0, offset)),
);
}
spans.add(
TextSpan(
text: remainingFullText.substring(
offset,
offset + token.length,
),
style: BotStyle.text(
context,
existingStyle: TextStyle(color: token.color(context)),
setColor: false,
),
// gesturRecognizer that sets selectedToken on click
// recognizer: TapGestureRecognizer()
// ..onTap = () {
// debugPrint('Token tapped');
// debugPrint(token.toJson().toString());
// if (mounted) {
// setState(() {
// if (selectedToken == token) {
// selectedToken = null;
// } else {
// selectedToken = token;
// }
// });
// }
// },
),
);
remainingFullText = remainingFullText.substring(offset + length);
}
if (remainingFullText.isNotEmpty) {
// Add any remaining text after the last token
spans.add(TextSpan(text: remainingFullText));
}
return TextSpan(children: spans);
} catch (err, s) {
ErrorHandler.logError(
e: err,
s: s,
data: {},
);
setState(() => error = err);
return const TextSpan(text: '');
}
}
@override
void initState() {
super.initState();
getSpeechToText().then((_) {
if (mounted) {
setState(() => transcriptText = _buildTranscriptText(context));
}
});
}
String? get wordsPerMinuteString =>
speechToTextResponse?.transcript.wordsPerMinute?.toStringAsFixed(2);
@override
Widget build(BuildContext context) {
if (_fetchingTranscription) {
return const ToolbarContentLoadingIndicator();
return const LinearProgressIndicator();
}
// done fetchig but not results means some kind of error
if (speechToTextResponse == null || error != null) {
return CardErrorWidget(
error: error ?? "Failed to fetch speech to text",
maxWidth: AppConfig.toolbarMinWidth,
// // done fetching but not results means some kind of error
if (_speechToTextResponse == null || error != null) {
return Row(
spacing: 8.0,
children: [
Flexible(
child: RichText(
text: TextSpan(
style: AppConfig.messageTextStyle(
widget.messageEvent.event,
widget.textColor,
),
children: [
WidgetSpan(
alignment: PlaceholderAlignment.middle,
child: Icon(
Icons.error,
color: Theme.of(context).colorScheme.error,
),
),
const TextSpan(text: " "),
TextSpan(
text: L10n.of(context).oopsSomethingWentWrong,
),
],
),
),
),
],
);
}
//TODO: find better icons
return ConstrainedBox(
constraints: const BoxConstraints(
maxWidth: AppConfig.toolbarMinWidth,
maxHeight: AppConfig.toolbarMaxHeight,
),
child: SingleChildScrollView(
child: Padding(
padding: const EdgeInsets.all(16),
child: Column(
mainAxisSize: MainAxisSize.min,
children: [
const SizedBox(height: 8),
RichText(
text: transcriptText!,
),
if (widget.messageEvent.senderId ==
Matrix.of(context).client.userID)
Column(
children: [
const SizedBox(height: 16),
Row(
mainAxisSize: MainAxisSize.min,
children: [
IconNumberWidget(
icon: Icons.speed,
number: wordsPerMinuteString != null
? "$wordsPerMinuteString"
: "??",
toolTip: L10n.of(context).wordsPerMinute,
),
],
),
const InstructionsInlineTooltip(
instructionsEnum: InstructionsEnum.speechToText,
),
],
),
],
),
),
return Text(
"${_speechToTextResponse?.transcript.text}",
style: AppConfig.messageTextStyle(
widget.messageEvent.event,
widget.textColor,
).copyWith(
fontStyle: FontStyle.italic,
),
);
}

26
lib/pangea/toolbar/widgets/overlay_message.dart
View file

@ -11,6 +11,7 @@ import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dar
import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart';
import 'package:fluffychat/pangea/toolbar/enums/reading_assistance_mode_enum.dart';
import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart';
import 'package:fluffychat/pangea/toolbar/widgets/message_speech_to_text_card.dart';
import 'package:fluffychat/utils/date_time_extension.dart';
import 'package:fluffychat/widgets/matrix.dart';
@ -127,6 +128,8 @@ class OverlayMessage extends StatelessWidget {
final showTranslation = overlayController.showTranslation &&
overlayController.translationText != null;
final showTranscription = pangeaMessageEvent?.isAudioMessage == true;
final content = Container(
decoration: BoxDecoration(
borderRadius: BorderRadius.circular(
@ -276,7 +279,7 @@ class OverlayMessage extends StatelessWidget {
},
)
: content,
if (showTranslation)
if (showTranscription || showTranslation)
SizedBox(
width: messageWidth,
child: Padding(
@ -286,13 +289,20 @@ class OverlayMessage extends StatelessWidget {
12.0,
12.0,
),
child: Text(
overlayController.translationText!,
style:
AppConfig.messageTextStyle(event, textColor).copyWith(
fontStyle: FontStyle.italic,
),
),
child: showTranscription
? MessageSpeechToTextCard(
messageEvent: pangeaMessageEvent!,
textColor: textColor,
)
: Text(
overlayController.translationText!,
style: AppConfig.messageTextStyle(
event,
textColor,
).copyWith(
fontStyle: FontStyle.italic,
),
),
),
),
],