reorganize pangea rep / pangea message event files
This commit is contained in:
ggurdin
parent
19d11994d6
commit
9cb155fcf1
9 changed files with 609 additions and 752 deletions
|
|
@ -2149,7 +2149,7 @@ class ChatController extends State<ChatPageWithRoom>
|
|||
ownMessage: true,
|
||||
);
|
||||
|
||||
final stt = await messageEvent.getSpeechToText(
|
||||
final stt = await messageEvent.requestSpeechToText(
|
||||
MatrixState.pangeaController.languageController.userL1?.langCodeShort ??
|
||||
LanguageKeys.unknownLanguage,
|
||||
MatrixState.pangeaController.languageController.userL2?.langCodeShort ??
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ class ActivitySummaryAnalyticsModel {
|
|||
}
|
||||
|
||||
void addMessageConstructs(PangeaMessageEvent event) {
|
||||
final uses = event.originalSent?.vocabAndMorphUses();
|
||||
final uses = event.originalSent?.vocabAndMorphUses;
|
||||
if (uses == null || uses.isEmpty) return;
|
||||
addConstructs(event.senderId, uses);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,13 +4,14 @@ import 'dart:ui';
|
|||
|
||||
import 'package:flutter/foundation.dart';
|
||||
|
||||
import 'package:async/async.dart';
|
||||
import 'package:collection/collection.dart';
|
||||
import 'package:matrix/matrix.dart';
|
||||
import 'package:sentry_flutter/sentry_flutter.dart';
|
||||
import 'package:matrix/matrix.dart' hide Result;
|
||||
|
||||
import 'package:fluffychat/pangea/choreographer/choreo_record_model.dart';
|
||||
import 'package:fluffychat/pangea/common/constants/model_keys.dart';
|
||||
import 'package:fluffychat/pangea/events/event_wrappers/pangea_representation_event.dart';
|
||||
import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart';
|
||||
import 'package:fluffychat/pangea/events/models/representation_content_model.dart';
|
||||
import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart';
|
||||
import 'package:fluffychat/pangea/events/repo/language_detection_repo.dart';
|
||||
|
|
@ -72,12 +73,11 @@ class PangeaMessageEvent {
|
|||
|
||||
bool get isAudioMessage => _event.messageType == MessageTypes.Audio;
|
||||
|
||||
String? get mimetype {
|
||||
if (!isAudioMessage) return null;
|
||||
final Map<String, dynamic>? info = _event.content.tryGetMap("info");
|
||||
if (info == null) return null;
|
||||
return info["mime_type"] ?? info["mimetype"];
|
||||
}
|
||||
String? get _l2Code =>
|
||||
MatrixState.pangeaController.languageController.activeL2Code();
|
||||
|
||||
String? get _l1Code =>
|
||||
MatrixState.pangeaController.languageController.userL1?.langCode;
|
||||
|
||||
Event? _latestEditCache;
|
||||
Event get _latestEdit => _latestEditCache ??= _event
|
||||
|
|
@ -92,131 +92,6 @@ class PangeaMessageEvent {
|
|||
.firstOrNull ??
|
||||
_event;
|
||||
|
||||
void updateLatestEdit() {
|
||||
_latestEditCache = null;
|
||||
_representations = null;
|
||||
}
|
||||
|
||||
Future<PangeaAudioFile> getMatrixAudioFile(
|
||||
String langCode,
|
||||
) async {
|
||||
final RepresentationEvent? rep = representationByLanguage(langCode);
|
||||
final tokensResp = await rep?.tokensGlobal(
|
||||
senderId,
|
||||
originServerTs,
|
||||
);
|
||||
|
||||
final TextToSpeechRequest params = TextToSpeechRequest(
|
||||
text: rep?.content.text ?? body,
|
||||
tokens: tokensResp?.result?.map((t) => t.text).toList() ?? [],
|
||||
langCode: langCode,
|
||||
userL1: l1Code ?? LanguageKeys.unknownLanguage,
|
||||
userL2: l2Code ?? LanguageKeys.unknownLanguage,
|
||||
);
|
||||
|
||||
final TextToSpeechResponse response =
|
||||
await MatrixState.pangeaController.textToSpeech.get(
|
||||
params,
|
||||
);
|
||||
|
||||
final audioBytes = base64.decode(response.audioContent);
|
||||
final eventIdParam = _event.eventId;
|
||||
final fileName =
|
||||
"audio_for_${eventIdParam}_$langCode.${response.fileExtension}";
|
||||
|
||||
final file = PangeaAudioFile(
|
||||
bytes: audioBytes,
|
||||
name: fileName,
|
||||
mimeType: response.mimeType,
|
||||
duration: response.durationMillis,
|
||||
waveform: response.waveform,
|
||||
tokens: response.ttsTokens,
|
||||
);
|
||||
|
||||
sendAudioEvent(file, response, rep?.text ?? body, langCode);
|
||||
|
||||
return file;
|
||||
}
|
||||
|
||||
Future<Event?> sendAudioEvent(
|
||||
PangeaAudioFile file,
|
||||
TextToSpeechResponse response,
|
||||
String text,
|
||||
String langCode,
|
||||
) async {
|
||||
final String? eventId = await room.sendFileEvent(
|
||||
file,
|
||||
inReplyTo: _event,
|
||||
extraContent: {
|
||||
'info': {
|
||||
...file.info,
|
||||
'duration': response.durationMillis,
|
||||
},
|
||||
'org.matrix.msc3245.voice': {},
|
||||
'org.matrix.msc1767.audio': {
|
||||
'duration': response.durationMillis,
|
||||
'waveform': response.waveform,
|
||||
},
|
||||
ModelKey.transcription:
|
||||
response.toPangeaAudioEventData(text, langCode).toJson(),
|
||||
},
|
||||
);
|
||||
|
||||
debugPrint("eventId in getTextToSpeechGlobal $eventId");
|
||||
final Event? audioEvent =
|
||||
eventId != null ? await room.getEventById(eventId) : null;
|
||||
|
||||
if (audioEvent == null) {
|
||||
return null;
|
||||
}
|
||||
allAudio.add(audioEvent);
|
||||
return audioEvent;
|
||||
}
|
||||
|
||||
Event? getTextToSpeechLocal(String langCode, String text) {
|
||||
return allAudio.firstWhereOrNull(
|
||||
(event) {
|
||||
try {
|
||||
// Safely access
|
||||
final dataMap = event.content.tryGetMap(ModelKey.transcription);
|
||||
|
||||
if (dataMap == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// old text to speech content will not have TTSToken data
|
||||
// we want to disregard them and just generate new ones
|
||||
// for that, we'll return false if 'tokens' are null
|
||||
// while in-development, we'll pause here to inspect
|
||||
// debugger can be removed after we're sure it's working
|
||||
if (dataMap['tokens'] == null) {
|
||||
// events before today will definitely not have the tokens
|
||||
debugger(
|
||||
when: kDebugMode &&
|
||||
event.originServerTs.isAfter(DateTime(2024, 10, 16)),
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
final PangeaAudioEventData audioData =
|
||||
PangeaAudioEventData.fromJson(dataMap as dynamic);
|
||||
|
||||
// Check if both language code and text match
|
||||
return audioData.langCode == langCode && audioData.text == text;
|
||||
} catch (e, s) {
|
||||
debugger(when: kDebugMode);
|
||||
ErrorHandler.logError(
|
||||
e: e,
|
||||
s: s,
|
||||
data: {},
|
||||
m: "error parsing data in getTextToSpeechLocal",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// get audio events that are related to this event
|
||||
Set<Event> get allAudio => _latestEdit
|
||||
.aggregatedEvents(
|
||||
|
|
@ -230,159 +105,25 @@ class PangeaMessageEvent {
|
|||
null;
|
||||
}).toSet();
|
||||
|
||||
SpeechToTextResponseModel? getSpeechToTextLocal() {
|
||||
final rawBotTranscription =
|
||||
event.content.tryGetMap(ModelKey.botTranscription);
|
||||
|
||||
if (rawBotTranscription != null) {
|
||||
try {
|
||||
return SpeechToTextResponseModel.fromJson(
|
||||
Map<String, dynamic>.from(rawBotTranscription),
|
||||
);
|
||||
} catch (err, s) {
|
||||
ErrorHandler.logError(
|
||||
e: err,
|
||||
s: s,
|
||||
data: {
|
||||
"event": _event.toJson(),
|
||||
},
|
||||
m: "error parsing botTranscription",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return representations
|
||||
.firstWhereOrNull(
|
||||
(element) => element.content.speechToText != null,
|
||||
)
|
||||
?.content
|
||||
.speechToText;
|
||||
}
|
||||
|
||||
Future<SpeechToTextResponseModel> getSpeechToText(
|
||||
String l1Code,
|
||||
String l2Code,
|
||||
) async {
|
||||
if (!isAudioMessage) {
|
||||
throw 'Calling getSpeechToText on non-audio message';
|
||||
}
|
||||
|
||||
final rawBotTranscription =
|
||||
event.content.tryGetMap(ModelKey.botTranscription);
|
||||
if (rawBotTranscription != null) {
|
||||
final SpeechToTextResponseModel botTranscription =
|
||||
SpeechToTextResponseModel.fromJson(
|
||||
Map<String, dynamic>.from(rawBotTranscription),
|
||||
);
|
||||
|
||||
_representations ??= [];
|
||||
_representations!.add(
|
||||
RepresentationEvent(
|
||||
timeline: timeline,
|
||||
parentMessageEvent: _event,
|
||||
content: PangeaRepresentation(
|
||||
langCode: botTranscription.langCode,
|
||||
text: botTranscription.transcript.text,
|
||||
originalSent: false,
|
||||
originalWritten: false,
|
||||
speechToText: botTranscription,
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
return botTranscription;
|
||||
}
|
||||
|
||||
final SpeechToTextResponseModel? speechToTextLocal = representations
|
||||
.firstWhereOrNull(
|
||||
(element) => element.content.speechToText != null,
|
||||
)
|
||||
?.content
|
||||
.speechToText;
|
||||
|
||||
if (speechToTextLocal != null) {
|
||||
return speechToTextLocal;
|
||||
}
|
||||
|
||||
final matrixFile = await _event.downloadAndDecryptAttachment();
|
||||
|
||||
final result = await SpeechToTextRepo.get(
|
||||
MatrixState.pangeaController.userController.accessToken,
|
||||
SpeechToTextRequestModel(
|
||||
audioContent: matrixFile.bytes,
|
||||
audioEvent: _event,
|
||||
config: SpeechToTextAudioConfigModel(
|
||||
encoding: mimeTypeToAudioEncoding(matrixFile.mimeType),
|
||||
sampleRateHertz: 22050,
|
||||
userL1: l1Code,
|
||||
userL2: l2Code,
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
if (result.error != null) {
|
||||
throw Exception(
|
||||
"Error getting speech to text: ${result.error}",
|
||||
);
|
||||
}
|
||||
|
||||
final SpeechToTextResponseModel response = result.result!;
|
||||
_representations?.add(
|
||||
RepresentationEvent(
|
||||
timeline: timeline,
|
||||
parentMessageEvent: _event,
|
||||
content: PangeaRepresentation(
|
||||
langCode: response.langCode,
|
||||
text: response.transcript.text,
|
||||
originalSent: false,
|
||||
originalWritten: false,
|
||||
speechToText: response,
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
Future<String> sttTranslationByLanguageGlobal({
|
||||
required String langCode,
|
||||
required String l1Code,
|
||||
required String l2Code,
|
||||
}) async {
|
||||
if (!representations.any(
|
||||
(element) => element.content.speechToText != null,
|
||||
)) {
|
||||
await getSpeechToText(l1Code, l2Code);
|
||||
}
|
||||
|
||||
final rep = representations.firstWhereOrNull(
|
||||
(element) => element.content.speechToText != null,
|
||||
);
|
||||
|
||||
if (rep == null) {
|
||||
throw Exception("No speech to text representation found");
|
||||
}
|
||||
|
||||
final resp = await rep.getSttTranslation(userL1: l1Code, userL2: l2Code);
|
||||
return resp.translation;
|
||||
}
|
||||
|
||||
PangeaMessageTokens? _tokensSafe(Map<String, dynamic>? content) {
|
||||
try {
|
||||
if (content == null) return null;
|
||||
return PangeaMessageTokens.fromJson(content);
|
||||
} catch (e, s) {
|
||||
debugger(when: kDebugMode);
|
||||
ErrorHandler.logError(
|
||||
e: e,
|
||||
s: s,
|
||||
data: content ?? {},
|
||||
m: "error parsing tokensSent",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
List<RepresentationEvent> get _repEvents => _latestEdit
|
||||
.aggregatedEvents(
|
||||
timeline,
|
||||
PangeaEventTypes.representation,
|
||||
)
|
||||
.map(
|
||||
(e) => RepresentationEvent(
|
||||
event: e,
|
||||
parentMessageEvent: _event,
|
||||
timeline: timeline,
|
||||
),
|
||||
)
|
||||
.sorted(
|
||||
(a, b) {
|
||||
if (a.event == null) return -1;
|
||||
if (b.event == null) return 1;
|
||||
return b.event!.originServerTs.compareTo(a.event!.originServerTs);
|
||||
},
|
||||
).toList();
|
||||
|
||||
ChoreoRecordModel? get _embeddedChoreo {
|
||||
try {
|
||||
|
|
@ -402,6 +143,22 @@ class PangeaMessageEvent {
|
|||
}
|
||||
}
|
||||
|
||||
PangeaMessageTokens? _tokensSafe(Map<String, dynamic>? content) {
|
||||
try {
|
||||
if (content == null) return null;
|
||||
return PangeaMessageTokens.fromJson(content);
|
||||
} catch (e, s) {
|
||||
debugger(when: kDebugMode);
|
||||
ErrorHandler.logError(
|
||||
e: e,
|
||||
s: s,
|
||||
data: content ?? {},
|
||||
m: "error parsing tokensSent",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
List<RepresentationEvent>? _representations;
|
||||
List<RepresentationEvent> get representations {
|
||||
if (_representations != null) return _representations!;
|
||||
|
|
@ -419,18 +176,6 @@ class PangeaMessageEvent {
|
|||
choreo: _embeddedChoreo,
|
||||
timeline: timeline,
|
||||
);
|
||||
if (_latestEdit.content[ModelKey.choreoRecord] == null) {
|
||||
Sentry.addBreadcrumb(
|
||||
Breadcrumb(
|
||||
message: "originalSent created without _event or _choreo",
|
||||
data: {
|
||||
"eventId": _latestEdit.eventId,
|
||||
"room": _latestEdit.room.id,
|
||||
"sender": _latestEdit.senderId,
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
// If originalSent has no tokens, there is not way to generate a tokens event
|
||||
// and send it as a related event, since original sent has not eventID to set
|
||||
|
|
@ -480,155 +225,10 @@ class PangeaMessageEvent {
|
|||
}
|
||||
}
|
||||
|
||||
_representations!.addAll(
|
||||
_latestEdit
|
||||
.aggregatedEvents(
|
||||
timeline,
|
||||
PangeaEventTypes.representation,
|
||||
)
|
||||
.map(
|
||||
(e) => RepresentationEvent(
|
||||
event: e,
|
||||
parentMessageEvent: _event,
|
||||
timeline: timeline,
|
||||
),
|
||||
)
|
||||
.sorted(
|
||||
(a, b) {
|
||||
//TODO - test with edited events to make sure this is working
|
||||
if (a.event == null) return -1;
|
||||
if (b.event == null) return 1;
|
||||
return b.event!.originServerTs.compareTo(a.event!.originServerTs);
|
||||
},
|
||||
).toList(),
|
||||
);
|
||||
|
||||
_representations!.addAll(_repEvents);
|
||||
return _representations!;
|
||||
}
|
||||
|
||||
RepresentationEvent? representationByLanguage(
|
||||
String langCode, {
|
||||
bool Function(RepresentationEvent)? filter,
|
||||
}) =>
|
||||
representations.firstWhereOrNull(
|
||||
(element) =>
|
||||
element.langCode.split("-")[0] == langCode.split("-")[0] &&
|
||||
(filter?.call(element) ?? true),
|
||||
);
|
||||
|
||||
Future<String?> representationByDetectedLanguage() async {
|
||||
LanguageDetectionResponse? resp;
|
||||
try {
|
||||
resp = await LanguageDetectionRepo.get(
|
||||
MatrixState.pangeaController.userController.accessToken,
|
||||
request: LanguageDetectionRequest(
|
||||
text: _latestEdit.body,
|
||||
senderl1: l1Code,
|
||||
senderl2: l2Code,
|
||||
),
|
||||
);
|
||||
} catch (e, s) {
|
||||
ErrorHandler.logError(
|
||||
e: e,
|
||||
s: s,
|
||||
data: {
|
||||
"event": _event.toJson(),
|
||||
},
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
final langCode = resp.detections.firstOrNull?.langCode;
|
||||
if (langCode == null) return null;
|
||||
if (langCode == originalSent?.langCode) {
|
||||
return originalSent?.event?.eventId;
|
||||
}
|
||||
|
||||
// clear representations cache so the new representation event can be added when next requested
|
||||
_representations = null;
|
||||
|
||||
final res = await FullTextTranslationRepo.get(
|
||||
MatrixState.pangeaController.userController.accessToken,
|
||||
FullTextTranslationRequestModel(
|
||||
text: originalSent?.content.text ?? _latestEdit.body,
|
||||
srcLang: originalSent?.langCode,
|
||||
tgtLang: langCode,
|
||||
userL2: l2Code ?? LanguageKeys.unknownLanguage,
|
||||
userL1: l1Code ?? LanguageKeys.unknownLanguage,
|
||||
),
|
||||
);
|
||||
|
||||
if (res.isError) return null;
|
||||
final repEvent = await room.sendPangeaEvent(
|
||||
content: PangeaRepresentation(
|
||||
langCode: langCode,
|
||||
text: res.result!,
|
||||
originalSent: originalSent == null,
|
||||
originalWritten: false,
|
||||
).toJson(),
|
||||
parentEventId: eventId,
|
||||
type: PangeaEventTypes.representation,
|
||||
);
|
||||
return repEvent?.eventId;
|
||||
}
|
||||
|
||||
Future<String> l1Respresentation() async {
|
||||
if (l1Code == null || l2Code == null) {
|
||||
throw Exception("Missing language codes");
|
||||
}
|
||||
|
||||
final includedIT =
|
||||
(originalSent?.choreo?.endedWithIT(originalSent!.text) ?? false) &&
|
||||
!(originalSent?.choreo?.includedIGC ?? true);
|
||||
|
||||
RepresentationEvent? rep;
|
||||
if (!includedIT) {
|
||||
// if the message didn't go through translation, get any l1 rep
|
||||
rep = representationByLanguage(l1Code!);
|
||||
} else {
|
||||
// if the message went through translation, get the non-original
|
||||
// l1 rep since originalWritten could contain some l2 words
|
||||
// (https://github.com/pangeachat/client/issues/3591)
|
||||
rep = representationByLanguage(
|
||||
l1Code!,
|
||||
filter: (rep) => !rep.content.originalWritten,
|
||||
);
|
||||
}
|
||||
|
||||
if (rep != null) return rep.content.text;
|
||||
|
||||
final String srcLang = includedIT
|
||||
? (originalWritten?.langCode ?? l1Code!)
|
||||
: (originalSent?.langCode ?? l2Code!);
|
||||
|
||||
// clear representations cache so the new representation event can be added when next requested
|
||||
_representations = null;
|
||||
|
||||
final resp = await FullTextTranslationRepo.get(
|
||||
MatrixState.pangeaController.userController.accessToken,
|
||||
FullTextTranslationRequestModel(
|
||||
text: includedIT ? originalWrittenContent : messageDisplayText,
|
||||
srcLang: srcLang,
|
||||
tgtLang: l1Code!,
|
||||
userL2: l2Code!,
|
||||
userL1: l1Code!,
|
||||
),
|
||||
);
|
||||
|
||||
if (resp.isError) throw resp.error!;
|
||||
room.sendPangeaEvent(
|
||||
content: PangeaRepresentation(
|
||||
langCode: l1Code!,
|
||||
text: resp.result!,
|
||||
originalSent: false,
|
||||
originalWritten: false,
|
||||
).toJson(),
|
||||
parentEventId: eventId,
|
||||
type: PangeaEventTypes.representation,
|
||||
);
|
||||
return resp.result!;
|
||||
}
|
||||
|
||||
RepresentationEvent? get originalSent => representations
|
||||
.firstWhereOrNull((element) => element.content.originalSent);
|
||||
|
||||
|
|
@ -646,12 +246,6 @@ class PangeaMessageEvent {
|
|||
return written ?? body;
|
||||
}
|
||||
|
||||
String? get l2Code =>
|
||||
MatrixState.pangeaController.languageController.activeL2Code();
|
||||
|
||||
String? get l1Code =>
|
||||
MatrixState.pangeaController.languageController.userL1?.langCode;
|
||||
|
||||
String get messageDisplayLangCode {
|
||||
if (isAudioMessage) {
|
||||
final stt = getSpeechToTextLocal();
|
||||
|
|
@ -665,7 +259,7 @@ class PangeaMessageEvent {
|
|||
|
||||
final String? originalLangCode = originalSent?.langCode;
|
||||
|
||||
final String? langCode = immersionMode ? l2Code : originalLangCode;
|
||||
final String? langCode = immersionMode ? _l2Code : originalLangCode;
|
||||
return langCode ?? LanguageKeys.unknownLanguage;
|
||||
}
|
||||
|
||||
|
|
@ -681,4 +275,319 @@ class PangeaMessageEvent {
|
|||
PLanguageStore.rtlLanguageCodes.contains(messageDisplayLangCode)
|
||||
? TextDirection.rtl
|
||||
: TextDirection.ltr;
|
||||
|
||||
void updateLatestEdit() {
|
||||
_latestEditCache = null;
|
||||
_representations = null;
|
||||
}
|
||||
|
||||
RepresentationEvent? representationByLanguage(
|
||||
String langCode, {
|
||||
bool Function(RepresentationEvent)? filter,
|
||||
}) =>
|
||||
representations.firstWhereOrNull(
|
||||
(element) =>
|
||||
element.langCode.split("-")[0] == langCode.split("-")[0] &&
|
||||
(filter?.call(element) ?? true),
|
||||
);
|
||||
|
||||
Event? getTextToSpeechLocal(String langCode, String text) {
|
||||
for (final audio in allAudio) {
|
||||
final dataMap = audio.content.tryGetMap(ModelKey.transcription);
|
||||
if (dataMap == null || !dataMap.containsKey('tokens')) continue;
|
||||
|
||||
try {
|
||||
final PangeaAudioEventData audioData = PangeaAudioEventData.fromJson(
|
||||
dataMap as dynamic,
|
||||
);
|
||||
|
||||
if (audioData.langCode == langCode && audioData.text == text) {
|
||||
return audio;
|
||||
}
|
||||
} catch (e, s) {
|
||||
debugger(when: kDebugMode);
|
||||
ErrorHandler.logError(
|
||||
e: e,
|
||||
s: s,
|
||||
data: {
|
||||
"event": audio.toJson(),
|
||||
},
|
||||
m: "error parsing data in getTextToSpeechLocal",
|
||||
);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
SpeechToTextResponseModel? getSpeechToTextLocal() {
|
||||
final rawBotTranscription =
|
||||
event.content.tryGetMap(ModelKey.botTranscription);
|
||||
|
||||
if (rawBotTranscription != null) {
|
||||
try {
|
||||
return SpeechToTextResponseModel.fromJson(
|
||||
Map<String, dynamic>.from(rawBotTranscription),
|
||||
);
|
||||
} catch (err, s) {
|
||||
ErrorHandler.logError(
|
||||
e: err,
|
||||
s: s,
|
||||
data: {
|
||||
"event": _event.toJson(),
|
||||
},
|
||||
m: "error parsing botTranscription",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return representations
|
||||
.firstWhereOrNull(
|
||||
(element) => element.content.speechToText != null,
|
||||
)
|
||||
?.content
|
||||
.speechToText;
|
||||
}
|
||||
|
||||
Future<PangeaAudioFile> requestTextToSpeech(
|
||||
String langCode,
|
||||
) async {
|
||||
final local = getTextToSpeechLocal(langCode, messageDisplayText);
|
||||
if (local != null) {
|
||||
final file = await local.getPangeaAudioFile();
|
||||
if (file != null) return file;
|
||||
}
|
||||
|
||||
final rep = representationByLanguage(langCode);
|
||||
final tokensResp = await rep?.requestTokens();
|
||||
final request = TextToSpeechRequest(
|
||||
text: rep?.content.text ?? body,
|
||||
tokens: tokensResp?.result?.map((t) => t.text).toList() ?? [],
|
||||
langCode: langCode,
|
||||
userL1: _l1Code ?? LanguageKeys.unknownLanguage,
|
||||
userL2: _l2Code ?? LanguageKeys.unknownLanguage,
|
||||
);
|
||||
|
||||
final response = await MatrixState.pangeaController.textToSpeech.get(
|
||||
request,
|
||||
);
|
||||
|
||||
final audioBytes = base64.decode(response.audioContent);
|
||||
final fileName =
|
||||
"audio_for_${_event.eventId}_$langCode.${response.fileExtension}";
|
||||
|
||||
final file = PangeaAudioFile(
|
||||
bytes: audioBytes,
|
||||
name: fileName,
|
||||
mimeType: response.mimeType,
|
||||
duration: response.durationMillis,
|
||||
waveform: response.waveform,
|
||||
tokens: response.ttsTokens,
|
||||
);
|
||||
|
||||
room.sendFileEvent(
|
||||
file,
|
||||
inReplyTo: _event,
|
||||
extraContent: {
|
||||
'info': {
|
||||
...file.info,
|
||||
'duration': response.durationMillis,
|
||||
},
|
||||
'org.matrix.msc3245.voice': {},
|
||||
'org.matrix.msc1767.audio': {
|
||||
'duration': response.durationMillis,
|
||||
'waveform': response.waveform,
|
||||
},
|
||||
ModelKey.transcription: response
|
||||
.toPangeaAudioEventData(rep?.text ?? body, langCode)
|
||||
.toJson(),
|
||||
},
|
||||
).then((eventId) async {
|
||||
final Event? audioEvent =
|
||||
eventId != null ? await room.getEventById(eventId) : null;
|
||||
|
||||
if (audioEvent != null) {
|
||||
allAudio.add(audioEvent);
|
||||
}
|
||||
});
|
||||
|
||||
return file;
|
||||
}
|
||||
|
||||
Future<SpeechToTextResponseModel> requestSpeechToText(
|
||||
String l1Code,
|
||||
String l2Code,
|
||||
) async {
|
||||
if (!isAudioMessage) {
|
||||
throw 'Calling getSpeechToText on non-audio message';
|
||||
}
|
||||
|
||||
final speechToTextLocal = getSpeechToTextLocal();
|
||||
if (speechToTextLocal != null) {
|
||||
return speechToTextLocal;
|
||||
}
|
||||
|
||||
final matrixFile = await _event.downloadAndDecryptAttachment();
|
||||
final result = await SpeechToTextRepo.get(
|
||||
MatrixState.pangeaController.userController.accessToken,
|
||||
SpeechToTextRequestModel(
|
||||
audioContent: matrixFile.bytes,
|
||||
audioEvent: _event,
|
||||
config: SpeechToTextAudioConfigModel(
|
||||
encoding: mimeTypeToAudioEncoding(matrixFile.mimeType),
|
||||
sampleRateHertz: 22050,
|
||||
userL1: l1Code,
|
||||
userL2: l2Code,
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
if (result.error != null) {
|
||||
throw Exception(
|
||||
"Error getting speech to text: ${result.error}",
|
||||
);
|
||||
}
|
||||
|
||||
_representations = null;
|
||||
return result.result!;
|
||||
}
|
||||
|
||||
Future<String> requestSttTranslation({
|
||||
required String langCode,
|
||||
required String l1Code,
|
||||
required String l2Code,
|
||||
}) async {
|
||||
if (!representations.any(
|
||||
(element) => element.content.speechToText != null,
|
||||
)) {
|
||||
await requestSpeechToText(l1Code, l2Code);
|
||||
}
|
||||
|
||||
final rep = representations.firstWhereOrNull(
|
||||
(element) => element.content.speechToText != null,
|
||||
);
|
||||
|
||||
if (rep == null) {
|
||||
throw Exception("No speech to text representation found");
|
||||
}
|
||||
|
||||
final resp = await rep.requestSttTranslation(
|
||||
userL1: l1Code,
|
||||
userL2: l2Code,
|
||||
);
|
||||
return resp.translation;
|
||||
}
|
||||
|
||||
Future<String?> requestRepresentationByDetectedLanguage() async {
|
||||
LanguageDetectionResponse? resp;
|
||||
final result = await LanguageDetectionRepo.get(
|
||||
MatrixState.pangeaController.userController.accessToken,
|
||||
LanguageDetectionRequest(
|
||||
text: _latestEdit.body,
|
||||
senderl1: _l1Code,
|
||||
senderl2: _l2Code,
|
||||
),
|
||||
);
|
||||
|
||||
if (result.isError) return null;
|
||||
resp = result.result!;
|
||||
|
||||
final langCode = resp.detections.firstOrNull?.langCode;
|
||||
if (langCode == null) return null;
|
||||
if (langCode == originalSent?.langCode) {
|
||||
return originalSent?.event?.eventId;
|
||||
}
|
||||
|
||||
final res = await _requestRepresentation(
|
||||
originalSent?.content.text ?? _latestEdit.body,
|
||||
langCode,
|
||||
originalSent?.langCode ?? LanguageKeys.unknownLanguage,
|
||||
originalSent: originalSent == null,
|
||||
);
|
||||
|
||||
if (res.isError) return null;
|
||||
return _sendRepresentationEvent(res.result!);
|
||||
}
|
||||
|
||||
Future<String> requestRespresentationByL1() async {
|
||||
if (_l1Code == null || _l2Code == null) {
|
||||
throw Exception("Missing language codes");
|
||||
}
|
||||
|
||||
final includedIT =
|
||||
(originalSent?.choreo?.endedWithIT(originalSent!.text) ?? false) &&
|
||||
!(originalSent?.choreo?.includedIGC ?? true);
|
||||
|
||||
RepresentationEvent? rep;
|
||||
if (!includedIT) {
|
||||
// if the message didn't go through translation, get any l1 rep
|
||||
rep = representationByLanguage(_l1Code!);
|
||||
} else {
|
||||
// if the message went through translation, get the non-original
|
||||
// l1 rep since originalWritten could contain some l2 words
|
||||
// (https://github.com/pangeachat/client/issues/3591)
|
||||
rep = representationByLanguage(
|
||||
_l1Code!,
|
||||
filter: (rep) => !rep.content.originalWritten,
|
||||
);
|
||||
}
|
||||
|
||||
if (rep != null) return rep.content.text;
|
||||
|
||||
final String srcLang = includedIT
|
||||
? (originalWritten?.langCode ?? _l1Code!)
|
||||
: (originalSent?.langCode ?? _l2Code!);
|
||||
|
||||
final resp = await _requestRepresentation(
|
||||
includedIT ? originalWrittenContent : messageDisplayText,
|
||||
_l1Code!,
|
||||
srcLang,
|
||||
);
|
||||
|
||||
if (resp.isError) throw resp.error!;
|
||||
_sendRepresentationEvent(resp.result!);
|
||||
return resp.result!.text;
|
||||
}
|
||||
|
||||
Future<Result<PangeaRepresentation>> _requestRepresentation(
|
||||
String text,
|
||||
String targetLang,
|
||||
String sourceLang, {
|
||||
bool originalSent = false,
|
||||
}) async {
|
||||
_representations = null;
|
||||
|
||||
final res = await FullTextTranslationRepo.get(
|
||||
MatrixState.pangeaController.userController.accessToken,
|
||||
FullTextTranslationRequestModel(
|
||||
text: text,
|
||||
srcLang: sourceLang,
|
||||
tgtLang: targetLang,
|
||||
userL2: _l2Code ?? LanguageKeys.unknownLanguage,
|
||||
userL1: _l1Code ?? LanguageKeys.unknownLanguage,
|
||||
),
|
||||
);
|
||||
|
||||
return res.isError
|
||||
? Result.error(res.error!)
|
||||
: Result.value(
|
||||
PangeaRepresentation(
|
||||
langCode: targetLang,
|
||||
text: res.result!,
|
||||
originalSent: originalSent,
|
||||
originalWritten: false,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
Future<String?> _sendRepresentationEvent(
|
||||
PangeaRepresentation representation,
|
||||
) async {
|
||||
final repEvent = await room.sendPangeaEvent(
|
||||
content: representation.toJson(),
|
||||
parentEventId: eventId,
|
||||
type: PangeaEventTypes.representation,
|
||||
);
|
||||
return repEvent?.eventId;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ import 'package:flutter/foundation.dart';
|
|||
import 'package:async/async.dart';
|
||||
import 'package:collection/collection.dart';
|
||||
import 'package:matrix/matrix.dart' hide Result;
|
||||
import 'package:matrix/src/utils/markdown.dart';
|
||||
import 'package:sentry_flutter/sentry_flutter.dart';
|
||||
|
||||
import 'package:fluffychat/pangea/analytics_misc/constructs_model.dart';
|
||||
|
|
@ -25,9 +24,6 @@ import 'package:fluffychat/pangea/events/repo/token_api_models.dart';
|
|||
import 'package:fluffychat/pangea/events/repo/tokens_repo.dart';
|
||||
import 'package:fluffychat/pangea/extensions/pangea_room_extension.dart';
|
||||
import 'package:fluffychat/pangea/learning_settings/constants/language_constants.dart';
|
||||
import 'package:fluffychat/pangea/morphs/morph_features_enum.dart';
|
||||
import 'package:fluffychat/pangea/morphs/parts_of_speech_enum.dart';
|
||||
import 'package:fluffychat/pangea/practice_activities/activity_type_enum.dart';
|
||||
import 'package:fluffychat/pangea/translation/full_text_translation_repo.dart';
|
||||
import 'package:fluffychat/pangea/translation/full_text_translation_request_model.dart';
|
||||
import 'package:fluffychat/widgets/future_loading_dialog.dart';
|
||||
|
|
@ -62,6 +58,33 @@ class RepresentationEvent {
|
|||
|
||||
Event? get event => _event;
|
||||
|
||||
String get text => content.text;
|
||||
|
||||
String get langCode => content.langCode;
|
||||
|
||||
List<LanguageDetectionModel>? get detections => _tokens?.detections;
|
||||
|
||||
Set<Event> get tokenEvents =>
|
||||
_event?.aggregatedEvents(
|
||||
timeline,
|
||||
PangeaEventTypes.tokens,
|
||||
) ??
|
||||
{};
|
||||
|
||||
Set<Event> get sttEvents =>
|
||||
_event?.aggregatedEvents(
|
||||
timeline,
|
||||
PangeaEventTypes.sttTranslation,
|
||||
) ??
|
||||
{};
|
||||
|
||||
Set<Event> get choreoEvents =>
|
||||
_event?.aggregatedEvents(
|
||||
timeline,
|
||||
PangeaEventTypes.choreoRecord,
|
||||
) ??
|
||||
{};
|
||||
|
||||
// Note: in the case where the event is the originalSent or originalWritten event,
|
||||
// the content will be set on initialization by the PangeaMessageEvent
|
||||
// Otherwise, the content will be fetched from the event where it is stored in content[type]
|
||||
|
|
@ -71,50 +94,134 @@ class RepresentationEvent {
|
|||
return _content!;
|
||||
}
|
||||
|
||||
String get text => content.text;
|
||||
|
||||
String get langCode => content.langCode;
|
||||
|
||||
bool get botAuthored =>
|
||||
content.originalSent == false && content.originalWritten == false;
|
||||
|
||||
List<LanguageDetectionModel>? get detections => _tokens?.detections;
|
||||
|
||||
List<PangeaToken>? get tokens {
|
||||
if (_tokens != null) return _tokens!.tokens;
|
||||
if (_event == null) return null;
|
||||
|
||||
final Set<Event> tokenEvents = _event?.aggregatedEvents(
|
||||
timeline,
|
||||
PangeaEventTypes.tokens,
|
||||
) ??
|
||||
{};
|
||||
|
||||
if (tokenEvents.isEmpty) return null;
|
||||
_tokens = tokenEvents.last.getPangeaContent<PangeaMessageTokens>();
|
||||
return _tokens?.tokens;
|
||||
}
|
||||
|
||||
Future<Result<List<PangeaToken>>> tokensGlobal(
|
||||
String senderID,
|
||||
DateTime timestamp,
|
||||
) async {
|
||||
if (tokens != null) return Result.value(tokens!);
|
||||
ChoreoRecordModel? get choreo {
|
||||
if (_choreo != null) return _choreo;
|
||||
|
||||
if (_event == null && timestamp.isAfter(DateTime(2024, 9, 25))) {
|
||||
if (_event == null) {
|
||||
Sentry.addBreadcrumb(
|
||||
Breadcrumb(
|
||||
message:
|
||||
'representation with no _event and no tokens got tokens directly. This means an original_sent with no tokens. This should not happen in messages sent after September 25',
|
||||
data: {
|
||||
'content': content.toJson(),
|
||||
'event': _event?.toJson(),
|
||||
'timestamp': timestamp.toIso8601String(),
|
||||
'senderID': senderID,
|
||||
},
|
||||
message: "_event and _choreo both null",
|
||||
),
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (choreoEvents.isEmpty) return null;
|
||||
if (choreoEvents.length > 1) {
|
||||
debugger(when: kDebugMode);
|
||||
ErrorHandler.logError(
|
||||
m: 'should not have more than one choreoEvent per representation ${_event?.eventId}',
|
||||
s: StackTrace.current,
|
||||
data: {"event": _event?.toJson()},
|
||||
);
|
||||
}
|
||||
|
||||
return ChoreoEvent(event: choreoEvents.first).content;
|
||||
}
|
||||
|
||||
List<SttTranslationModel> get sttTranslations {
|
||||
if (content.speechToText == null) return [];
|
||||
if (_event == null) {
|
||||
Sentry.addBreadcrumb(
|
||||
Breadcrumb(
|
||||
message: "_event and _sttTranslations both null",
|
||||
),
|
||||
);
|
||||
return [];
|
||||
}
|
||||
|
||||
if (sttEvents.isEmpty) return [];
|
||||
final List<SttTranslationModel> sttTranslations = [];
|
||||
for (final event in sttEvents) {
|
||||
try {
|
||||
sttTranslations.add(
|
||||
SttTranslationModel.fromJson(event.content),
|
||||
);
|
||||
} catch (e) {
|
||||
Sentry.addBreadcrumb(
|
||||
Breadcrumb(
|
||||
message: "Failed to parse STT translation",
|
||||
data: {
|
||||
"eventID": event.eventId,
|
||||
"content": event.content,
|
||||
"error": e.toString(),
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
return sttTranslations;
|
||||
}
|
||||
|
||||
List<OneConstructUse> get vocabAndMorphUses {
|
||||
if (tokens == null || tokens!.isEmpty) {
|
||||
return [];
|
||||
}
|
||||
|
||||
final metadata = ConstructUseMetaData(
|
||||
roomId: parentMessageEvent.room.id,
|
||||
timeStamp: parentMessageEvent.originServerTs,
|
||||
eventId: parentMessageEvent.eventId,
|
||||
);
|
||||
|
||||
return content.vocabAndMorphUses(
|
||||
tokens: tokens!,
|
||||
metadata: metadata,
|
||||
choreo: choreo,
|
||||
);
|
||||
}
|
||||
|
||||
/// Finds the closest non-punctuation token to the given token.
|
||||
PangeaToken? getClosestNonPunctToken(PangeaToken token) {
|
||||
// If it's not punctuation, it's already the closest.
|
||||
if (token.pos != "PUNCT") return token;
|
||||
|
||||
final list = tokens;
|
||||
if (list == null) return null;
|
||||
|
||||
final index = list.indexOf(token);
|
||||
if (index == -1) return null;
|
||||
|
||||
PangeaToken? left;
|
||||
PangeaToken? right;
|
||||
|
||||
// Scan left
|
||||
for (int i = index - 1; i >= 0; i--) {
|
||||
if (list[i].pos != "PUNCT") {
|
||||
left = list[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Scan right
|
||||
for (int i = index + 1; i < list.length; i++) {
|
||||
if (list[i].pos != "PUNCT") {
|
||||
right = list[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (left == null) return right;
|
||||
if (right == null) return left;
|
||||
|
||||
// Choose the nearest by distance
|
||||
final leftDistance = token.start - left.end;
|
||||
final rightDistance = right.start - token.end;
|
||||
|
||||
return leftDistance < rightDistance ? left : right;
|
||||
}
|
||||
|
||||
Future<Result<List<PangeaToken>>> requestTokens() async {
|
||||
if (tokens != null) return Result.value(tokens!);
|
||||
final res = await TokensRepo.get(
|
||||
MatrixState.pangeaController.userController.accessToken,
|
||||
TokensRequestModel(
|
||||
|
|
@ -140,90 +247,12 @@ class RepresentationEvent {
|
|||
);
|
||||
}
|
||||
|
||||
if (res.isError) {
|
||||
return Result.error(res.error!);
|
||||
} else {
|
||||
return Result.value(res.result!.tokens);
|
||||
}
|
||||
return res.isError
|
||||
? Result.error(res.error!)
|
||||
: Result.value(res.result!.tokens);
|
||||
}
|
||||
|
||||
Future<void> sendTokensEvent(
|
||||
String repEventID,
|
||||
Room room,
|
||||
String userl1,
|
||||
String userl2,
|
||||
) async {
|
||||
if (tokens != null) return;
|
||||
if (_event == null) {
|
||||
ErrorHandler.logError(
|
||||
e: "Called getTokensEvent with no _event",
|
||||
data: {},
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
final resp = await TokensRepo.get(
|
||||
MatrixState.pangeaController.userController.accessToken,
|
||||
TokensRequestModel(
|
||||
fullText: text,
|
||||
langCode: langCode,
|
||||
senderL1: userl1,
|
||||
senderL2: userl2,
|
||||
),
|
||||
);
|
||||
|
||||
if (resp.isError) return;
|
||||
room.sendPangeaEvent(
|
||||
content: PangeaMessageTokens(
|
||||
tokens: resp.result!.tokens,
|
||||
detections: resp.result!.detections,
|
||||
).toJson(),
|
||||
parentEventId: _event!.eventId,
|
||||
type: PangeaEventTypes.tokens,
|
||||
);
|
||||
}
|
||||
|
||||
List<SttTranslationModel> get sttTranslations {
|
||||
if (content.speechToText == null) return [];
|
||||
if (_event == null) {
|
||||
Sentry.addBreadcrumb(
|
||||
Breadcrumb(
|
||||
message: "_event and _sttTranslations both null",
|
||||
),
|
||||
);
|
||||
return [];
|
||||
}
|
||||
|
||||
final Set<Event> sttEvents = _event!.aggregatedEvents(
|
||||
timeline,
|
||||
PangeaEventTypes.sttTranslation,
|
||||
);
|
||||
|
||||
if (sttEvents.isEmpty) return [];
|
||||
final List<SttTranslationModel> sttTranslations = [];
|
||||
for (final event in sttEvents) {
|
||||
try {
|
||||
sttTranslations.add(
|
||||
SttTranslationModel.fromJson(event.content),
|
||||
);
|
||||
} catch (e) {
|
||||
Sentry.addBreadcrumb(
|
||||
Breadcrumb(
|
||||
message: "Failed to parse STT translation",
|
||||
data: {
|
||||
"eventID": event.eventId,
|
||||
"content": event.content,
|
||||
"error": e.toString(),
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return sttTranslations;
|
||||
}
|
||||
|
||||
Future<SttTranslationModel> getSttTranslation({
|
||||
Future<SttTranslationModel> requestSttTranslation({
|
||||
required String userL1,
|
||||
required String userL2,
|
||||
}) async {
|
||||
|
|
@ -263,137 +292,4 @@ class RepresentationEvent {
|
|||
|
||||
return translation;
|
||||
}
|
||||
|
||||
ChoreoRecordModel? get choreo {
|
||||
if (_choreo != null) return _choreo;
|
||||
|
||||
if (_event == null) {
|
||||
Sentry.addBreadcrumb(
|
||||
Breadcrumb(
|
||||
message: "_event and _choreo both null",
|
||||
),
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
final Set<Event> choreoMatrixEvents =
|
||||
_event?.aggregatedEvents(timeline, PangeaEventTypes.choreoRecord) ?? {};
|
||||
|
||||
if (choreoMatrixEvents.isEmpty) return null;
|
||||
|
||||
if (choreoMatrixEvents.length > 1) {
|
||||
debugger(when: kDebugMode);
|
||||
ErrorHandler.logError(
|
||||
m: 'should not have more than one choreoEvent per representation ${_event?.eventId}',
|
||||
s: StackTrace.current,
|
||||
data: {"event": _event?.toJson()},
|
||||
);
|
||||
}
|
||||
|
||||
_choreo = ChoreoEvent(event: choreoMatrixEvents.first).content;
|
||||
|
||||
return _choreo;
|
||||
}
|
||||
|
||||
String? formatBody() {
|
||||
return markdown(content.text);
|
||||
}
|
||||
|
||||
/// Finds the closest non-punctuation token to the given token.
|
||||
///
|
||||
/// This method checks if the provided token is a punctuation token. If it is not,
|
||||
/// it returns the token itself. If the token is a punctuation token, it searches
|
||||
/// through the list of tokens to find the closest non-punctuation token either to
|
||||
/// the left or right of the given token.
|
||||
///
|
||||
/// If both left and right non-punctuation tokens are found, it returns the one
|
||||
/// that is closest to the given token. If only one of them is found, it returns
|
||||
/// that token. If no non-punctuation tokens are found, it returns null.
|
||||
///
|
||||
/// - Parameters:
|
||||
/// - token: The token for which to find the closest non-punctuation token.
|
||||
///
|
||||
/// - Returns: The closest non-punctuation token, or null if no such token exists.
|
||||
PangeaToken? getClosestNonPunctToken(PangeaToken token) {
|
||||
if (token.pos != "PUNCT") return token;
|
||||
if (tokens == null) return null;
|
||||
final index = tokens!.indexOf(token);
|
||||
if (index > -1) {
|
||||
final leftTokens = tokens!.sublist(0, index);
|
||||
final rightTokens = tokens!.sublist(index + 1);
|
||||
final leftMostToken = leftTokens.lastWhereOrNull(
|
||||
(element) => element.pos != "PUNCT",
|
||||
);
|
||||
final rightMostToken = rightTokens.firstWhereOrNull(
|
||||
(element) => element.pos != "PUNCT",
|
||||
);
|
||||
|
||||
if (leftMostToken != null && rightMostToken != null) {
|
||||
final leftDistance = token.start - leftMostToken.end;
|
||||
final rightDistance = rightMostToken.start - token.end;
|
||||
return leftDistance < rightDistance ? leftMostToken : rightMostToken;
|
||||
} else if (leftMostToken != null) {
|
||||
return leftMostToken;
|
||||
} else if (rightMostToken != null) {
|
||||
return rightMostToken;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
List<PangeaToken> get tokensToSave =>
|
||||
tokens?.where((token) => token.lemma.saveVocab).toList() ?? [];
|
||||
|
||||
// List<ConstructIdentifier> get allTokenMorphsToConstructIdentifiers => tokens?.map((t) => t.morphConstructIds).toList() ??
|
||||
// [];
|
||||
|
||||
/// get allTokenMorphsToConstructIdentifiers
|
||||
Set<MorphFeaturesEnum> get morphFeatureSetToPractice =>
|
||||
MorphFeaturesEnum.values.where((feature) {
|
||||
// pos is always included
|
||||
if (feature == MorphFeaturesEnum.Pos) {
|
||||
return true;
|
||||
}
|
||||
return tokens?.any((token) => token.morph.containsKey(feature)) ??
|
||||
false;
|
||||
}).toSet();
|
||||
|
||||
Set<PartOfSpeechEnum> posSetToPractice(ActivityTypeEnum a) =>
|
||||
PartOfSpeechEnum.values.where((pos) {
|
||||
// some pos are not eligible for practice at all
|
||||
if (!pos.eligibleForPractice(a)) {
|
||||
return false;
|
||||
}
|
||||
return tokens?.any(
|
||||
(token) => token.pos.toLowerCase() == pos.name.toLowerCase(),
|
||||
) ??
|
||||
false;
|
||||
}).toSet();
|
||||
|
||||
List<String> tagsByFeature(MorphFeaturesEnum feature) {
|
||||
return tokens
|
||||
?.where((t) => t.morph.containsKey(feature))
|
||||
.map((t) => t.morph[feature])
|
||||
.cast<String>()
|
||||
.toList() ??
|
||||
[];
|
||||
}
|
||||
|
||||
List<OneConstructUse> vocabAndMorphUses() {
|
||||
if (tokens == null || tokens!.isEmpty) {
|
||||
return [];
|
||||
}
|
||||
|
||||
final metadata = ConstructUseMetaData(
|
||||
roomId: parentMessageEvent.room.id,
|
||||
timeStamp: parentMessageEvent.originServerTs,
|
||||
eventId: parentMessageEvent.eventId,
|
||||
);
|
||||
|
||||
return content.vocabAndMorphUses(
|
||||
tokens: tokens!,
|
||||
metadata: metadata,
|
||||
choreo: choreo,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,18 +1,47 @@
|
|||
import 'dart:convert';
|
||||
|
||||
import 'package:async/async.dart';
|
||||
import 'package:http/http.dart';
|
||||
|
||||
import 'package:fluffychat/pangea/common/config/environment.dart';
|
||||
import 'package:fluffychat/pangea/common/network/requests.dart';
|
||||
import 'package:fluffychat/pangea/common/network/urls.dart';
|
||||
import 'package:fluffychat/pangea/common/utils/error_handler.dart';
|
||||
import 'package:fluffychat/pangea/events/repo/language_detection_request.dart';
|
||||
import 'package:fluffychat/pangea/events/repo/language_detection_response.dart';
|
||||
|
||||
class _LanguageDetectionCacheItem {
|
||||
final Future<LanguageDetectionResponse> data;
|
||||
final DateTime timestamp;
|
||||
|
||||
const _LanguageDetectionCacheItem({
|
||||
required this.data,
|
||||
required this.timestamp,
|
||||
});
|
||||
}
|
||||
|
||||
class LanguageDetectionRepo {
|
||||
static Future<LanguageDetectionResponse> get(
|
||||
String? accessToken, {
|
||||
required LanguageDetectionRequest request,
|
||||
}) async {
|
||||
static final Map<String, _LanguageDetectionCacheItem> _cache = {};
|
||||
static const Duration _cacheDuration = Duration(minutes: 10);
|
||||
|
||||
static Future<Result<LanguageDetectionResponse>> get(
|
||||
String accessToken,
|
||||
LanguageDetectionRequest request,
|
||||
) {
|
||||
final cached = _getCached(request);
|
||||
if (cached != null) {
|
||||
return _getResult(request, cached);
|
||||
}
|
||||
|
||||
final future = _fetch(accessToken, request);
|
||||
_setCached(request, future);
|
||||
return _getResult(request, future);
|
||||
}
|
||||
|
||||
static Future<LanguageDetectionResponse> _fetch(
|
||||
String accessToken,
|
||||
LanguageDetectionRequest request,
|
||||
) async {
|
||||
final Requests req = Requests(
|
||||
accessToken: accessToken,
|
||||
choreoApiKey: Environment.choreoApiKey,
|
||||
|
|
@ -22,12 +51,54 @@ class LanguageDetectionRepo {
|
|||
body: request.toJson(),
|
||||
);
|
||||
|
||||
final Map<String, dynamic> json =
|
||||
jsonDecode(utf8.decode(res.bodyBytes).toString());
|
||||
if (res.statusCode != 200) {
|
||||
throw Exception(
|
||||
'Failed to detect language: ${res.statusCode} ${res.reasonPhrase}',
|
||||
);
|
||||
}
|
||||
|
||||
final LanguageDetectionResponse response =
|
||||
LanguageDetectionResponse.fromJson(json);
|
||||
|
||||
return response;
|
||||
return LanguageDetectionResponse.fromJson(
|
||||
jsonDecode(utf8.decode(res.bodyBytes)),
|
||||
);
|
||||
}
|
||||
|
||||
static Future<Result<LanguageDetectionResponse>> _getResult(
|
||||
LanguageDetectionRequest request,
|
||||
Future<LanguageDetectionResponse> future,
|
||||
) async {
|
||||
try {
|
||||
final res = await future;
|
||||
return Result.value(res);
|
||||
} catch (e, s) {
|
||||
_cache.remove(request.hashCode.toString());
|
||||
ErrorHandler.logError(
|
||||
e: e,
|
||||
s: s,
|
||||
data: request.toJson(),
|
||||
);
|
||||
return Result.error(e);
|
||||
}
|
||||
}
|
||||
|
||||
static Future<LanguageDetectionResponse>? _getCached(
|
||||
LanguageDetectionRequest request,
|
||||
) {
|
||||
final cacheKeys = [..._cache.keys];
|
||||
for (final key in cacheKeys) {
|
||||
if (DateTime.now().difference(_cache[key]!.timestamp) >= _cacheDuration) {
|
||||
_cache.remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
return _cache[request.hashCode.toString()]?.data;
|
||||
}
|
||||
|
||||
static void _setCached(
|
||||
LanguageDetectionRequest request,
|
||||
Future<LanguageDetectionResponse> response,
|
||||
) =>
|
||||
_cache[request.hashCode.toString()] = _LanguageDetectionCacheItem(
|
||||
data: response,
|
||||
timestamp: DateTime.now(),
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,4 +16,16 @@ class LanguageDetectionRequest {
|
|||
'sender_l2': senderl2,
|
||||
};
|
||||
}
|
||||
|
||||
@override
|
||||
int get hashCode => text.hashCode ^ senderl1.hashCode ^ senderl2.hashCode;
|
||||
|
||||
@override
|
||||
bool operator ==(Object other) {
|
||||
if (identical(this, other)) return true;
|
||||
return other is LanguageDetectionRequest &&
|
||||
other.text == text &&
|
||||
other.senderl1 == senderl1 &&
|
||||
other.senderl2 == senderl2;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ import 'package:fluffychat/pages/chat/events/audio_player.dart';
|
|||
import 'package:fluffychat/pangea/analytics_misc/text_loading_shimmer.dart';
|
||||
import 'package:fluffychat/pangea/common/utils/error_handler.dart';
|
||||
import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart';
|
||||
import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart';
|
||||
import 'package:fluffychat/pangea/toolbar/controllers/text_to_speech_controller.dart';
|
||||
|
||||
class MessageAudioCard extends StatefulWidget {
|
||||
|
|
@ -42,19 +41,9 @@ class MessageAudioCardState extends State<MessageAudioCard> {
|
|||
setState(() => _isLoading = true);
|
||||
|
||||
try {
|
||||
final String langCode = widget.messageEvent.messageDisplayLangCode;
|
||||
final Event? localEvent = widget.messageEvent.getTextToSpeechLocal(
|
||||
langCode,
|
||||
widget.messageEvent.messageDisplayText,
|
||||
audioFile = await widget.messageEvent.requestTextToSpeech(
|
||||
widget.messageEvent.messageDisplayLangCode,
|
||||
);
|
||||
|
||||
if (localEvent != null) {
|
||||
audioFile = await localEvent.getPangeaAudioFile();
|
||||
} else {
|
||||
audioFile = await widget.messageEvent.getMatrixAudioFile(
|
||||
langCode,
|
||||
);
|
||||
}
|
||||
debugPrint("audio file is now: $audioFile. setting starts and ends...");
|
||||
if (mounted) setState(() => _isLoading = false);
|
||||
} catch (e, s) {
|
||||
|
|
|
|||
|
|
@ -112,12 +112,7 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
|
|||
}
|
||||
|
||||
if (repEvent?.event != null) {
|
||||
await repEvent!.sendTokensEvent(
|
||||
repEvent.event!.eventId,
|
||||
widget._event.room,
|
||||
MatrixState.pangeaController.languageController.userL1!.langCode,
|
||||
MatrixState.pangeaController.languageController.userL2!.langCode,
|
||||
);
|
||||
await repEvent!.requestTokens();
|
||||
}
|
||||
} catch (e, s) {
|
||||
debugger(when: kDebugMode);
|
||||
|
|
@ -268,7 +263,8 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
|
|||
pangeaMessageEvent.messageDisplayRepresentation;
|
||||
|
||||
if (repEvent != null) return repEvent;
|
||||
final eventID = await pangeaMessageEvent.representationByDetectedLanguage();
|
||||
final eventID =
|
||||
await pangeaMessageEvent.requestRepresentationByDetectedLanguage();
|
||||
|
||||
if (eventID == null) return null;
|
||||
final event = await widget._event.room.getEventById(eventID);
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import 'package:path_provider/path_provider.dart';
|
|||
|
||||
import 'package:fluffychat/pangea/common/utils/async_state.dart';
|
||||
import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart';
|
||||
import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart';
|
||||
import 'package:fluffychat/pangea/speech_to_text/speech_to_text_response_model.dart';
|
||||
import 'package:fluffychat/pangea/toolbar/widgets/message_audio_card.dart';
|
||||
import 'package:fluffychat/pangea/toolbar/widgets/select_mode_buttons.dart';
|
||||
|
|
@ -19,7 +18,7 @@ class _TranscriptionLoader extends AsyncLoader<SpeechToTextResponseModel> {
|
|||
_TranscriptionLoader(this.messageEvent) : super();
|
||||
|
||||
@override
|
||||
Future<SpeechToTextResponseModel> fetch() => messageEvent.getSpeechToText(
|
||||
Future<SpeechToTextResponseModel> fetch() => messageEvent.requestSpeechToText(
|
||||
MatrixState.pangeaController.languageController.userL1!.langCodeShort,
|
||||
MatrixState.pangeaController.languageController.userL2!.langCodeShort,
|
||||
);
|
||||
|
|
@ -30,7 +29,7 @@ class _STTTranslationLoader extends AsyncLoader<String> {
|
|||
_STTTranslationLoader(this.messageEvent) : super();
|
||||
|
||||
@override
|
||||
Future<String> fetch() => messageEvent.sttTranslationByLanguageGlobal(
|
||||
Future<String> fetch() => messageEvent.requestSttTranslation(
|
||||
langCode: MatrixState
|
||||
.pangeaController.languageController.userL1!.langCodeShort,
|
||||
l1Code: MatrixState
|
||||
|
|
@ -45,7 +44,7 @@ class _TranslationLoader extends AsyncLoader<String> {
|
|||
_TranslationLoader(this.messageEvent) : super();
|
||||
|
||||
@override
|
||||
Future<String> fetch() => messageEvent.l1Respresentation();
|
||||
Future<String> fetch() => messageEvent.requestRespresentationByL1();
|
||||
}
|
||||
|
||||
class _AudioLoader extends AsyncLoader<(PangeaAudioFile, File?)> {
|
||||
|
|
@ -54,25 +53,10 @@ class _AudioLoader extends AsyncLoader<(PangeaAudioFile, File?)> {
|
|||
|
||||
@override
|
||||
Future<(PangeaAudioFile, File?)> fetch() async {
|
||||
final String langCode = messageEvent.messageDisplayLangCode;
|
||||
|
||||
final Event? localEvent = messageEvent.getTextToSpeechLocal(
|
||||
langCode,
|
||||
messageEvent.messageDisplayText,
|
||||
final audioBytes = await messageEvent.requestTextToSpeech(
|
||||
messageEvent.messageDisplayLangCode,
|
||||
);
|
||||
|
||||
PangeaAudioFile? audioBytes;
|
||||
if (localEvent != null) {
|
||||
audioBytes = await localEvent.getPangeaAudioFile();
|
||||
} else {
|
||||
audioBytes = await messageEvent.getMatrixAudioFile(
|
||||
langCode,
|
||||
);
|
||||
}
|
||||
if (audioBytes == null) {
|
||||
throw Exception('Audio bytes are null');
|
||||
}
|
||||
|
||||
File? audioFile;
|
||||
if (!kIsWeb) {
|
||||
final tempDir = await getTemporaryDirectory();
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue