ember/fluffychat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

now displaying score for pronunciation

Browse source
This commit is contained in:
William Jordan-Cooley 2024-05-07 16:10:53 -04:00
parent 6cfdd349bd
commit 191fc69628
6 changed files with 455 additions and 120 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
assets/l10n/intl_en.arb
View file

@ -3935,5 +3935,9 @@
"unread": {}
}
},
"messageAnalytics": "Message Analytics"
"messageAnalytics": "Message Analytics",
"words": "Words",
"score": "Score",
"accuracy": "Accuracy",
"points": "Points"
}

61
lib/pangea/models/speech_to_text_models.dart
View file

@ -1,6 +1,5 @@
import 'dart:convert';
import 'package:fluffychat/config/app_config.dart';
import 'package:fluffychat/pangea/enum/audio_encoding_enum.dart';
import 'package:fluffychat/pangea/models/pangea_token_model.dart';
import 'package:flutter/foundation.dart';
@ -88,10 +87,14 @@ class STTToken {
int get length => token.text.length;
Color color(BuildContext context) {
if (confidence == null || confidence! > 80) {
return Theme.of(context).brightness == Brightness.dark
? AppConfig.primaryColorLight
: AppConfig.primaryColor;
if (confidence == null) {
return Theme.of(context).textTheme.bodyMedium?.color ??
(Theme.of(context).brightness == Brightness.dark
? Colors.white
: Colors.black);
}
if (confidence! > 80) {
return const Color.fromARGB(255, 0, 152, 0);
}
if (confidence! > 50) {
return const Color.fromARGB(255, 184, 142, 43);
@ -99,16 +102,19 @@ class STTToken {
return Colors.red;
}
factory STTToken.fromJson(Map<String, dynamic> json) => STTToken(
token: PangeaToken.fromJson(json['token']),
startTime: json['start_time'] != null
? Duration(milliseconds: json['start_time'])
: null,
endTime: json['end_time'] != null
? Duration(milliseconds: json['end_time'])
: null,
confidence: json['confidence'],
);
factory STTToken.fromJson(Map<String, dynamic> json) {
// debugPrint('STTToken.fromJson: $json');
return STTToken(
token: PangeaToken.fromJson(json['token']),
startTime: json['start_time'] != null
? Duration(milliseconds: json['start_time'] * 1000.toInt())
: null,
endTime: json['end_time'] != null
? Duration(milliseconds: json['end_time'] * 1000.toInt())
: null,
confidence: json['confidence'],
);
}
Map<String, dynamic> toJson() => {
"token": token,
@ -116,6 +122,27 @@ class STTToken {
"end_time": endTime?.inMilliseconds,
"confidence": confidence,
};
@override
bool operator ==(Object other) {
if (identical(this, other)) return true;
if (other is! STTToken) return false;
return token == other.token &&
startTime == other.startTime &&
endTime == other.endTime &&
confidence == other.confidence;
}
@override
int get hashCode {
return Object.hashAll([
token.hashCode,
startTime.hashCode,
endTime.hashCode,
confidence.hashCode,
]);
}
}
class Transcript {
@ -133,7 +160,9 @@ class Transcript {
factory Transcript.fromJson(Map<String, dynamic> json) => Transcript(
text: json['transcript'],
confidence: json['confidence'].toDouble(),
confidence: json['confidence'] <= 100
? json['confidence']
: json['confidence'] / 100,
sttTokens: (json['stt_tokens'] as List)
.map((e) => STTToken.fromJson(e))
.toList(),

144
lib/pangea/widgets/chat/message_speech_to_text_card.dart
View file

@ -1,12 +1,14 @@
import 'package:fluffychat/pangea/matrix_event_wrappers/pangea_message_event.dart';
import 'package:fluffychat/pangea/models/speech_to_text_models.dart';
import 'package:fluffychat/pangea/utils/error_handler.dart';
import 'package:fluffychat/pangea/widgets/chat/speech_to_text_score.dart';
import 'package:fluffychat/pangea/widgets/chat/speech_to_text_text.dart';
import 'package:fluffychat/pangea/widgets/chat/toolbar_content_loading_indicator.dart';
import 'package:fluffychat/pangea/widgets/common/icon_number_widget.dart';
import 'package:fluffychat/pangea/widgets/igc/card_error_widget.dart';
import 'package:fluffychat/widgets/matrix.dart';
import 'package:flutter/gestures.dart';
import 'package:flutter/material.dart';
import 'package:flutter_gen/gen_l10n/l10n.dart';
import '../../utils/bot_style.dart';
class MessageSpeechToTextCard extends StatefulWidget {
final PangeaMessageEvent messageEvent;
@ -24,6 +26,7 @@ class MessageSpeechToTextCardState extends State<MessageSpeechToTextCard> {
SpeechToTextModel? speechToTextResponse;
bool _fetchingTranscription = true;
Object? error;
STTToken? selectedToken;
String? get l1Code =>
MatrixState.pangeaController.languageController.activeL1Code(
@ -37,26 +40,90 @@ class MessageSpeechToTextCardState extends State<MessageSpeechToTextCard> {
// look for transcription in message event
// if not found, call API to transcribe audio
Future<void> getSpeechToText() async {
try {
if (l1Code == null || l2Code == null) {
throw Exception('Language selection not found');
}
speechToTextResponse ??=
await widget.messageEvent.getSpeechToText(l1Code!, l2Code!);
debugPrint(
'Speech to text transcript: ${speechToTextResponse?.transcript.text}',
);
} catch (e, s) {
error = e;
ErrorHandler.logError(
e: e,
s: s,
data: widget.messageEvent.event.content,
);
} finally {
setState(() => _fetchingTranscription = false);
// try {
if (l1Code == null || l2Code == null) {
throw Exception('Language selection not found');
}
speechToTextResponse ??=
await widget.messageEvent.getSpeechToText(l1Code!, l2Code!);
debugPrint(
'Speech to text transcript: ${speechToTextResponse?.transcript.text}',
);
// } catch (e, s) {
// debugger(when: kDebugMode);
// error = e;
// ErrorHandler.logError(
// e: e,
// s: s,
// data: widget.messageEvent.event.content,
// );
// } finally {
setState(() => _fetchingTranscription = false);
// }
}
TextSpan _buildTranscriptText(BuildContext context) {
final Transcript transcript = speechToTextResponse!.transcript;
final List<InlineSpan> spans = [];
final String fullText = transcript.text;
int lastEnd = 0;
for (final token in transcript.sttTokens) {
// debugPrint('Token confidence: ${token.confidence}');
// debugPrint('color: ${token.color(context)}');
if (token.offset > lastEnd) {
// Add any plain text before the token
spans.add(
TextSpan(
text: fullText.substring(lastEnd, token.offset),
),
);
// debugPrint('Pre: ${fullText.substring(lastEnd, token.offset)}');
}
spans.add(
TextSpan(
text: fullText.substring(token.offset, token.offset + token.length),
style: BotStyle.text(
context,
existingStyle: TextStyle(color: token.color(context)),
setColor: false,
),
// gesturRecognizer that sets selectedToken on click
recognizer: TapGestureRecognizer()
..onTap = () {
debugPrint('Token tapped');
debugPrint(token.toJson().toString());
setState(() {
if (selectedToken == token) {
selectedToken = null;
} else {
selectedToken = token;
}
});
},
),
);
// debugPrint(
// 'Main: ${fullText.substring(token.offset, token.offset + token.length)}',
// );
lastEnd = token.offset + token.length;
}
if (lastEnd < fullText.length) {
// Add any remaining text after the last token
spans.add(
TextSpan(
text: fullText.substring(lastEnd),
),
);
// debugPrint('Post: ${fullText.substring(lastEnd)}');
}
return TextSpan(children: spans);
}
@override
@ -76,12 +143,37 @@ class MessageSpeechToTextCardState extends State<MessageSpeechToTextCard> {
return CardErrorWidget(error: error);
}
final int words = speechToTextResponse!.transcript.sttTokens.length;
final int accuracy = speechToTextResponse!.transcript.confidence;
final int total = words * accuracy;
//TODO: find better icons
return Column(
children: [
SpeechToTextText(transcript: speechToTextResponse!.transcript),
const Divider(),
SpeechToTextScoreWidget(
score: speechToTextResponse!.transcript.confidence,
RichText(
text: _buildTranscriptText(context),
),
const SizedBox(height: 16),
Row(
mainAxisAlignment: MainAxisAlignment.spaceEvenly,
children: [
IconNumberWidget(
icon: Icons.abc,
number: (selectedToken == null ? words : 1).toString(),
toolTip: L10n.of(context)!.words,
),
IconNumberWidget(
icon: Icons.approval,
number:
"${selectedToken?.confidence ?? speechToTextResponse!.transcript.confidence}%",
toolTip: L10n.of(context)!.accuracy,
),
IconNumberWidget(
icon: Icons.score,
number: (selectedToken?.confidence ?? total).toString(),
toolTip: L10n.of(context)!.points,
),
],
),
],
);

29
lib/pangea/widgets/chat/speech_to_text_score.dart
View file

@ -1,29 +0,0 @@
import 'package:flutter/material.dart';
class SpeechToTextScoreWidget extends StatefulWidget {
final int score;
const SpeechToTextScoreWidget({super.key, required this.score});
@override
SpeechToTextScoreWidgetState createState() => SpeechToTextScoreWidgetState();
}
class SpeechToTextScoreWidgetState extends State<SpeechToTextScoreWidget> {
@override
Widget build(BuildContext context) {
return Padding(
padding: const EdgeInsets.symmetric(vertical: 8.0),
child: AnimatedSwitcher(
duration: const Duration(milliseconds: 500),
transitionBuilder: (Widget child, Animation<double> animation) {
return FadeTransition(opacity: animation, child: child);
},
child: Text(
'Score: ${widget.score}',
key: ValueKey<int>(widget.score),
style: const TextStyle(fontSize: 24, fontWeight: FontWeight.bold),
),
),
);
}
}

47
lib/pangea/widgets/common/icon_number_widget.dart Normal file
View file

@ -0,0 +1,47 @@
import 'package:flutter/material.dart';
class IconNumberWidget extends StatelessWidget {
final IconData icon;
final String number;
final Color? iconColor;
final double? iconSize;
final String? toolTip;
const IconNumberWidget({
super.key,
required this.icon,
required this.number,
this.toolTip,
this.iconColor,
this.iconSize,
});
Widget _content(BuildContext context) {
return Row(
mainAxisSize: MainAxisSize.min,
children: <Widget>[
Icon(
icon,
color: iconColor ?? Theme.of(context).iconTheme.color,
size: iconSize ?? Theme.of(context).iconTheme.size,
),
const SizedBox(width: 8),
Text(
number.toString(),
style: TextStyle(
fontSize:
iconSize ?? Theme.of(context).textTheme.bodyMedium?.fontSize,
color: iconColor ?? Theme.of(context).textTheme.bodyMedium?.color,
),
),
],
);
}
@override
Widget build(BuildContext context) {
return toolTip != null
? Tooltip(message: toolTip!, child: _content(context))
: _content(context);
}
}

288
needed-translations.txt
View file

@ -814,7 +814,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"be": [
@ -2227,7 +2231,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"bn": [
@ -3102,7 +3110,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"bo": [
@ -3977,7 +3989,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"ca": [
@ -4852,7 +4868,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"cs": [
@ -5727,7 +5747,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"de": [
@ -6549,7 +6573,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"el": [
@ -7424,7 +7452,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"eo": [
@ -8299,7 +8331,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"es": [
@ -8322,7 +8358,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"et": [
@ -9140,7 +9180,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"eu": [
@ -9958,7 +10002,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"fa": [
@ -10833,7 +10881,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"fi": [
@ -11708,7 +11760,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"fr": [
@ -12583,7 +12639,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"ga": [
@ -13458,7 +13518,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"gl": [
@ -14276,7 +14340,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"he": [
@ -15151,7 +15219,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"hi": [
@ -16026,7 +16098,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"hr": [
@ -16888,7 +16964,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"hu": [
@ -17763,7 +17843,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"ia": [
@ -19162,7 +19246,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"id": [
@ -20037,7 +20125,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"ie": [
@ -20912,7 +21004,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"it": [
@ -21772,7 +21868,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"ja": [
@ -22647,7 +22747,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"ko": [
@ -23522,7 +23626,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"lt": [
@ -24397,7 +24505,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"lv": [
@ -25272,7 +25384,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"nb": [
@ -26147,7 +26263,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"nl": [
@ -27022,7 +27142,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"pl": [
@ -27897,7 +28021,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"pt": [
@ -28772,7 +28900,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"pt_BR": [
@ -29616,7 +29748,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"pt_PT": [
@ -30491,7 +30627,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"ro": [
@ -31366,7 +31506,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"ru": [
@ -32184,7 +32328,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"sk": [
@ -33059,7 +33207,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"sl": [
@ -33934,7 +34086,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"sr": [
@ -34809,7 +34965,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"sv": [
@ -35649,7 +35809,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"ta": [
@ -36524,7 +36688,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"th": [
@ -37399,7 +37567,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"tr": [
@ -38259,7 +38431,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"uk": [
@ -39077,7 +39253,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"vi": [
@ -39952,7 +40132,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"zh": [
@ -40770,7 +40954,11 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
],
"zh_Hant": [
@ -41645,6 +41833,10 @@
"commandHint_ignore",
"commandHint_unignore",
"unreadChatsInApp",
"messageAnalytics"
"messageAnalytics",
"words",
"score",
"accuracy",
"points"
]
}