ember/fluffychat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #3182 from pangeachat/3168-allow-clicking-of-words-in-transcripts

Browse source 
feat: make tokens in STT transcript clickable
This commit is contained in:
ggurdin 2025-06-20 15:03:29 -04:00 • committed by GitHub
commit 12e66cfcdc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 190 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
lib/pangea/analytics_details_popup/vocab_analytics_details_view.dart
View file

@ -28,9 +28,6 @@ class VocabDetailsView extends StatelessWidget {
ConstructUses get _construct => constructId.constructUses;
String? get _userL1 =>
MatrixState.pangeaController.languageController.userL1?.langCode;
/// Get the language code for the current lemma
String? get _userL2 =>
MatrixState.pangeaController.languageController.userL2?.langCode;

9
lib/pangea/choreographer/controllers/igc_controller.dart
View file

@ -256,14 +256,7 @@ class IgcController {
timeline: choreographer.chatController.timeline!,
ownMessage: event.senderId ==
choreographer.pangeaController.matrixState.client.userID,
)
.getSpeechToTextLocal(
choreographer.l1LangCode,
choreographer.l2LangCode,
)
?.transcript
.text
.trim(); // trim whitespace
).getSpeechToTextLocal()?.transcript.text.trim(); // trim whitespace
if (content == null) continue;
messages.add(
PreviousMessage(

8
lib/pangea/events/event_wrappers/pangea_message_event.dart
View file

@ -232,13 +232,7 @@ class PangeaMessageEvent {
null;
}).toSet();
SpeechToTextModel? getSpeechToTextLocal(
String? l1Code,
String? l2Code,
) {
if (l1Code == null || l2Code == null) {
return null;
}
SpeechToTextModel? getSpeechToTextLocal() {
return representations
.firstWhereOrNull(
(element) => element.content.speechToText != null,

2
lib/pangea/lemmas/lemma_reaction_picker.dart
View file

@ -53,7 +53,7 @@ class LemmaReactionPickerState extends State<LemmaReactionPicker> {
} catch (e, s) {
ErrorHandler.logError(data: widget.cId.toJson(), e: e, s: s);
} finally {
setState(() => loading = false);
if (mounted) setState(() => loading = false);
}
}

100
lib/pangea/message_token_text/token_position_model.dart
View file

@ -1,29 +1,85 @@
import 'package:fluffychat/pangea/events/models/pangea_token_model.dart';
class TokenPositionModel {
/// Start index of the full substring in the message
final int start;
/// End index of the full substring in the message
final int end;
/// Start index of the token in the message
final int tokenStart;
/// End index of the token in the message
final int tokenEnd;
final bool selected;
final bool hideContent;
class TokenPosition {
final PangeaToken? token;
final int startIndex;
final int endIndex;
const TokenPositionModel({
required this.start,
required this.end,
required this.tokenStart,
required this.tokenEnd,
required this.hideContent,
required this.selected,
const TokenPosition({
this.token,
required this.startIndex,
required this.endIndex,
});
}
class TokensUtil {
static List<TokenPosition> getTokenPositions(
List<PangeaToken> tokens,
) {
final List<TokenPosition> tokenPositions = [];
int tokenPointer = 0;
int globalPointer = 0;
while (tokenPointer < tokens.length) {
int endIndex = tokenPointer;
PangeaToken token = tokens[tokenPointer];
if (token.text.offset > globalPointer) {
// If the token starts after the current global pointer, we need to
// create a new token position for the gap
tokenPositions.add(
TokenPosition(
startIndex: globalPointer,
endIndex: token.text.offset,
),
);
globalPointer = token.text.offset;
}
// move the end index if the next token is right next to the current token
// and either the current token is punctuation or the next token is punctuation
while (endIndex < tokens.length - 1) {
final PangeaToken currentToken = tokens[endIndex];
final PangeaToken nextToken = tokens[endIndex + 1];
final currentIsPunct = currentToken.pos == 'PUNCT' &&
currentToken.text.content.trim().isNotEmpty;
final nextIsPunct = nextToken.pos == 'PUNCT' &&
nextToken.text.content.trim().isNotEmpty;
if (currentToken.text.offset + currentToken.text.length !=
nextToken.text.offset) {
break;
}
if ((currentIsPunct && nextIsPunct) ||
(currentIsPunct && nextToken.text.content.trim().isNotEmpty) ||
(nextIsPunct && currentToken.text.content.trim().isNotEmpty)) {
if (token.pos == 'PUNCT' && !nextIsPunct) {
token = nextToken;
}
endIndex++;
} else {
break;
}
}
tokenPositions.add(
TokenPosition(
token: token,
startIndex: tokens[tokenPointer].text.offset,
endIndex: tokens[endIndex].text.offset + tokens[endIndex].text.length,
),
);
// Move to the next token
tokenPointer = tokenPointer + (endIndex - tokenPointer) + 1;
globalPointer =
tokens[endIndex].text.offset + tokens[endIndex].text.length;
}
return tokenPositions;
}
}

15
lib/pangea/toolbar/widgets/message_selection_overlay.dart
View file

@ -461,9 +461,18 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
pangeaMessageEvent?.messageDisplayLangCode.split("-")[0] ==
MatrixState.pangeaController.languageController.userL2?.langCodeShort;
PangeaToken? get selectedToken =>
pangeaMessageEvent?.messageDisplayRepresentation?.tokens
?.firstWhereOrNull(isTokenSelected);
PangeaToken? get selectedToken {
if (pangeaMessageEvent?.isAudioMessage == true) {
final stt = pangeaMessageEvent!.getSpeechToTextLocal();
if (stt == null || stt.transcript.sttTokens.isEmpty) return null;
return stt.transcript.sttTokens
.firstWhereOrNull((t) => isTokenSelected(t.token))
?.token;
}
return pangeaMessageEvent?.messageDisplayRepresentation?.tokens
?.firstWhereOrNull(isTokenSelected);
}
/// Whether the overlay is currently displaying a selection
bool get isSelection => _selectedSpan != null || _highlightedTokens != null;

9
lib/pangea/toolbar/widgets/overlay_message.dart
View file

@ -15,6 +15,7 @@ import 'package:fluffychat/pangea/learning_settings/utils/p_language_store.dart'
import 'package:fluffychat/pangea/phonetic_transcription/phonetic_transcription_widget.dart';
import 'package:fluffychat/pangea/toolbar/enums/reading_assistance_mode_enum.dart';
import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart';
import 'package:fluffychat/pangea/toolbar/widgets/stt_transcript_tokens.dart';
import 'package:fluffychat/utils/date_time_extension.dart';
import 'package:fluffychat/utils/file_description.dart';
import 'package:fluffychat/widgets/matrix.dart';
@ -177,15 +178,17 @@ class OverlayMessage extends StatelessWidget {
spacing: 8.0,
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Text(
overlayController
.transcription!.transcript.text,
SttTranscriptTokens(
model: overlayController.transcription!,
style: AppConfig.messageTextStyle(
event,
textColor,
).copyWith(
fontStyle: FontStyle.italic,
),
onClick: overlayController
.onClickOverlayMessageToken,
isSelected: overlayController.isTokenSelected,
),
if (MatrixState.pangeaController
.languageController.showTrancription)

91
lib/pangea/toolbar/widgets/stt_transcript_tokens.dart Normal file
View file

@ -0,0 +1,91 @@
import 'package:flutter/material.dart';
import 'package:fluffychat/pangea/events/models/pangea_token_model.dart';
import 'package:fluffychat/pangea/message_token_text/token_position_model.dart';
import 'package:fluffychat/pangea/toolbar/models/speech_to_text_models.dart';
import 'package:fluffychat/widgets/matrix.dart';
class SttTranscriptTokens extends StatelessWidget {
final SpeechToTextModel model;
final TextStyle? style;
final void Function(PangeaToken)? onClick;
final bool Function(PangeaToken)? isSelected;
const SttTranscriptTokens({
super.key,
required this.model,
this.onClick,
this.isSelected,
this.style,
});
List<PangeaToken> get tokens =>
model.transcript.sttTokens.map((t) => t.token).toList();
@override
Widget build(BuildContext context) {
if (model.transcript.sttTokens.isEmpty) {
return Text(
model.transcript.text,
style: style ?? DefaultTextStyle.of(context).style,
textScaler: TextScaler.noScaling,
);
}
final messageCharacters = model.transcript.text.characters;
return RichText(
textScaler: TextScaler.noScaling,
text: TextSpan(
style: style ?? DefaultTextStyle.of(context).style,
children: TokensUtil.getTokenPositions(tokens).map((tokenPosition) {
final text = messageCharacters
.skip(tokenPosition.startIndex)
.take(tokenPosition.endIndex - tokenPosition.startIndex)
.toString();
if (tokenPosition.token == null) {
return TextSpan(
text: text,
style: style ?? DefaultTextStyle.of(context).style,
);
}
final token = tokenPosition.token!;
final selected = isSelected?.call(token) ?? false;
return WidgetSpan(
child: CompositedTransformTarget(
link: MatrixState.pAnyState
.layerLinkAndKey(token.text.uniqueKey)
.link,
child: MouseRegion(
key: MatrixState.pAnyState
.layerLinkAndKey(token.text.uniqueKey)
.key,
cursor: SystemMouseCursors.click,
child: GestureDetector(
behavior: HitTestBehavior.translucent,
onTap: onClick != null ? () => onClick?.call(token) : null,
child: RichText(
text: TextSpan(
text: text,
style: (style ?? DefaultTextStyle.of(context).style)
.copyWith(
decoration: TextDecoration.underline,
decorationThickness: 4,
decorationColor: selected
? Theme.of(context).colorScheme.primary
: Colors.white.withAlpha(0),
),
),
),
),
),
),
);
}).toList(),
),
);
}
}