chore: update tokens repo

lib/pangea/chat_list/utils/get_chat_list_item_subtitle.dart

@@ -7,7 +7,6 @@ import 'package:fluffychat/l10n/l10n.dart';
 import 'package:fluffychat/pangea/activity_sessions/activity_room_extension.dart';
 import 'package:fluffychat/pangea/course_chats/open_roles_indicator.dart';
 import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart';
-import 'package:fluffychat/pangea/events/models/pangea_token_model.dart';
 import 'package:fluffychat/widgets/matrix.dart';
 import '../../../utils/matrix_sdk_extensions/matrix_locals.dart';
 
@@ -29,29 +28,18 @@ class ChatListItemSubtitle extends StatelessWidget {
         !(AppConfig.renderHtml && !event.redacted && event.isRichMessage);
   }
 
-  Future<MessageEventAndTokens> _getPangeaMessageEvent(
+  Future<PangeaMessageEvent> _getPangeaMessageEvent(
     final Event event,
   ) async {
     final Timeline timeline = event.room.timeline != null
         ? event.room.timeline!
         : await event.room.getTimeline();
 
-    final pangeaMessageEvent = PangeaMessageEvent(
+    return PangeaMessageEvent(
       event: event,
       timeline: timeline,
       ownMessage: event.senderId == event.room.client.userID,
     );
-
-    final tokens =
-        await pangeaMessageEvent.messageDisplayRepresentation?.tokensGlobal(
-      event.senderId,
-      event.originServerTs,
-    );
-
-    return MessageEventAndTokens(
-      event: pangeaMessageEvent,
-      tokens: tokens,
-    );
   }
 
   @override
@@ -109,12 +97,11 @@ class ChatListItemSubtitle extends StatelessWidget {
       );
     }
 
-    return FutureBuilder(
+    return FutureBuilder<PangeaMessageEvent>(
       future: _getPangeaMessageEvent(event),
       builder: (context, snapshot) {
         if (snapshot.hasData) {
-          final messageEventAndTokens = snapshot.data as MessageEventAndTokens;
-          final pangeaMessageEvent = messageEventAndTokens.event;
+          final pangeaMessageEvent = snapshot.data!;
           return Text(
             pangeaMessageEvent.messageDisplayText,
             style: style,
@@ -133,13 +120,3 @@ class ChatListItemSubtitle extends StatelessWidget {
     );
   }
 }
-
-class MessageEventAndTokens {
-  final PangeaMessageEvent event;
-  final List<PangeaToken>? tokens;
-
-  MessageEventAndTokens({
-    required this.event,
-    required this.tokens,
-  });
-}

lib/pangea/choreographer/controllers/choreographer.dart

@@ -25,6 +25,7 @@ import 'package:fluffychat/pangea/extensions/pangea_room_extension.dart';
 import 'package:fluffychat/pangea/learning_settings/constants/language_constants.dart';
 import 'package:fluffychat/pangea/subscription/controllers/subscription_controller.dart';
 import 'package:fluffychat/pangea/toolbar/controllers/tts_controller.dart';
+import 'package:fluffychat/widgets/future_loading_dialog.dart';
 import '../../../widgets/matrix.dart';
 import 'error_service.dart';
 import 'it_controller.dart';
@@ -300,72 +301,54 @@ class Choreographer extends ChangeNotifier {
 
     final message = chatController.sendController.text;
     final fakeEventId = chatController.sendFakeMessage();
-    final PangeaRepresentation? originalWritten =
-        _choreoRecord?.includedIT == true &&
-                itController.sourceText.value != null
-            ? PangeaRepresentation(
-                langCode: l1LangCode ?? LanguageKeys.unknownLanguage,
-                text: itController.sourceText.value!,
-                originalWritten: true,
-                originalSent: false,
-              )
-            : null;
 
-    PangeaMessageTokens? tokensSent;
-    PangeaRepresentation? originalSent;
-    try {
-      TokensResponseModel? res;
-      if (l1LangCode != null && l2LangCode != null) {
-        res = await pangeaController.messageData
-            .getTokens(
-              repEventId: null,
-              room: chatController.room,
-              req: TokensRequestModel(
-                fullText: message,
-                senderL1: l1LangCode!,
-                senderL2: l2LangCode!,
-              ),
-            )
-            .timeout(const Duration(seconds: 10));
-      }
+    TokensResponseModel? tokensResp;
+    if (l1LangCode != null && l2LangCode != null) {
+      final res = await pangeaController.messageData
+          .getTokens(
+            repEventId: null,
+            room: chatController.room,
+            req: TokensRequestModel(
+              fullText: message,
+              senderL1: l1LangCode!,
+              senderL2: l2LangCode!,
+            ),
+          )
+          .timeout(const Duration(seconds: 10));
+      tokensResp = res.isValue ? res.result : null;
+    }
 
-      originalSent = PangeaRepresentation(
-        langCode: res?.detections.firstOrNull?.langCode ??
+    final hasOriginalWritten = _choreoRecord?.includedIT == true &&
+        itController.sourceText.value != null;
+
+    chatController.send(
+      message: message,
+      originalSent: PangeaRepresentation(
+        langCode: tokensResp?.detections.firstOrNull?.langCode ??
             LanguageKeys.unknownLanguage,
         text: message,
         originalSent: true,
-        originalWritten: originalWritten == null,
-      );
-
-      tokensSent = res != null
-          ? PangeaMessageTokens(
-              tokens: res.tokens,
-              detections: res.detections,
+        originalWritten: hasOriginalWritten,
+      ),
+      originalWritten: hasOriginalWritten
+          ? PangeaRepresentation(
+              langCode: l1LangCode ?? LanguageKeys.unknownLanguage,
+              text: itController.sourceText.value!,
+              originalWritten: true,
+              originalSent: false,
             )
-          : null;
-    } catch (e, s) {
-      ErrorHandler.logError(
-        e: e,
-        s: s,
-        data: {
-          "currentText": message,
-          "l1LangCode": l1LangCode,
-          "l2LangCode": l2LangCode,
-          "choreoRecord": _choreoRecord?.toJson(),
-        },
-        level: e is TimeoutException ? SentryLevel.warning : SentryLevel.error,
-      );
-    } finally {
-      chatController.send(
-        message: message,
-        originalSent: originalSent,
-        originalWritten: originalWritten,
-        tokensSent: tokensSent,
-        choreo: _choreoRecord,
-        tempEventId: fakeEventId,
-      );
-      clear();
-    }
+          : null,
+      tokensSent: tokensResp != null
+          ? PangeaMessageTokens(
+              tokens: tokensResp.tokens,
+              detections: tokensResp.detections,
+            )
+          : null,
+      choreo: _choreoRecord,
+      tempEventId: fakeEventId,
+    );
+
+    clear();
   }
 
   void openIT(PangeaMatchState itMatch) {

lib/pangea/events/controllers/message_data_controller.dart

@@ -1,9 +1,7 @@
 import 'dart:async';
 
-import 'package:flutter/foundation.dart';
-import 'package:flutter/material.dart';
-
-import 'package:matrix/matrix.dart';
+import 'package:async/async.dart';
+import 'package:matrix/matrix.dart' hide Result;
 
 import 'package:fluffychat/pangea/choreographer/repo/full_text_translation_repo.dart';
 import 'package:fluffychat/pangea/choreographer/repo/full_text_translation_request_model.dart';
@@ -19,56 +17,34 @@ import 'package:fluffychat/pangea/events/repo/token_api_models.dart';
 import 'package:fluffychat/pangea/events/repo/tokens_repo.dart';
 import 'package:fluffychat/pangea/extensions/pangea_room_extension.dart';
 import 'package:fluffychat/widgets/future_loading_dialog.dart';
+import 'package:fluffychat/widgets/matrix.dart';
 
 // TODO - make this static and take it out of the _pangeaController
 // will need to pass accessToken to the requests
 class MessageDataController extends BaseController {
   late PangeaController _pangeaController;
 
-  final Map<int, Future<TokensResponseModel>> _tokensCache = {};
-  late Timer _cacheTimer;
-
   MessageDataController(PangeaController pangeaController) {
     _pangeaController = pangeaController;
-    _startCacheTimer();
-  }
-
-  /// Starts a timer that clears the cache every 10 minutes
-  void _startCacheTimer() {
-    _cacheTimer = Timer.periodic(const Duration(minutes: 10), (timer) {
-      _clearCache();
-    });
-  }
-
-  /// Clears the token and representation caches
-  void _clearCache() {
-    _tokensCache.clear();
-    debugPrint("message data cache cleared.");
-  }
-
-  @override
-  void dispose() {
-    _cacheTimer.cancel(); // Cancel the timer when the controller is disposed
-    super.dispose();
   }
 
   /// get tokens from the server
   /// if repEventId is not null, send the tokens to the room
-  Future<TokensResponseModel> _getTokens({
+  Future<Result<TokensResponseModel>> getTokens({
     required String? repEventId,
     required TokensRequestModel req,
     required Room? room,
   }) async {
-    final TokensResponseModel res = await TokensRepo.get(
-      _pangeaController.userController.accessToken,
-      request: req,
+    final res = await TokensRepo.get(
+      MatrixState.pangeaController.userController.accessToken,
+      req,
     );
-    if (repEventId != null && room != null) {
+    if (res.isValue && repEventId != null && room != null) {
       room
           .sendPangeaEvent(
             content: PangeaMessageTokens(
-              tokens: res.tokens,
-              detections: res.detections,
+              tokens: res.result!.tokens,
+              detections: res.result!.detections,
             ).toJson(),
             parentEventId: repEventId,
             type: PangeaEventTypes.tokens,
@@ -82,27 +58,9 @@ class MessageDataController extends BaseController {
             ),
           );
     }
-
     return res;
   }
 
-  /// get tokens from the server
-  /// first check if the tokens are in the cache
-  /// if repEventId is not null, send the tokens to the room
-  Future<TokensResponseModel> getTokens({
-    required String? repEventId,
-    required TokensRequestModel req,
-    required Room? room,
-  }) =>
-      _tokensCache[req.hashCode] ??= _getTokens(
-        repEventId: repEventId,
-        req: req,
-        room: room,
-      ).catchError((e, s) {
-        _tokensCache.remove(req.hashCode);
-        return Future<TokensResponseModel>.error(e, s);
-      });
-
   /////// translation ////////
 
   /// get translation from the server
@@ -196,35 +154,6 @@ class MessageDataController extends BaseController {
     }
   }
 
-  Future<void> sendTokensEvent({
-    required String repEventId,
-    required TokensRequestModel req,
-    required Room room,
-  }) async {
-    final TokensResponseModel res = await TokensRepo.get(
-      _pangeaController.userController.accessToken,
-      request: req,
-    );
-
-    try {
-      await room.sendPangeaEvent(
-        content: PangeaMessageTokens(
-          tokens: res.tokens,
-          detections: res.detections,
-        ).toJson(),
-        parentEventId: repEventId,
-        type: PangeaEventTypes.tokens,
-      );
-    } catch (e, s) {
-      ErrorHandler.logError(
-        m: "error in _getTokens.sendPangeaEvent",
-        e: e,
-        s: s,
-        data: req.toJson(),
-      );
-    }
-  }
-
   Future<SttTranslationModel> getSttTranslation({
     required String? repEventId,
     required FullTextTranslationRequestModel req,

lib/pangea/events/event_wrappers/pangea_message_event.dart

@@ -28,6 +28,7 @@ import 'package:fluffychat/pangea/toolbar/enums/audio_encoding_enum.dart';
 import 'package:fluffychat/pangea/toolbar/event_wrappers/practice_activity_event.dart';
 import 'package:fluffychat/pangea/toolbar/models/speech_to_text_models.dart';
 import 'package:fluffychat/pangea/toolbar/widgets/message_audio_card.dart';
+import 'package:fluffychat/widgets/future_loading_dialog.dart';
 import '../../../widgets/matrix.dart';
 import '../../common/utils/error_handler.dart';
 import '../../learning_settings/constants/language_constants.dart';
@@ -97,20 +98,18 @@ class PangeaMessageEvent {
     _representations = null;
   }
 
-  Future<PangeaAudioFile?> getMatrixAudioFile(
+  Future<PangeaAudioFile> getMatrixAudioFile(
     String langCode,
   ) async {
     final RepresentationEvent? rep = representationByLanguage(langCode);
+    final tokensResp = await rep?.tokensGlobal(
+      senderId,
+      originServerTs,
+    );
 
     final TextToSpeechRequest params = TextToSpeechRequest(
       text: rep?.content.text ?? body,
-      tokens: (await rep?.tokensGlobal(
-            senderId,
-            originServerTs,
-          ))
-              ?.map((t) => t.text)
-              .toList() ??
-          [],
+      tokens: tokensResp?.result?.map((t) => t.text).toList() ?? [],
       langCode: langCode,
       userL1: l1Code ?? LanguageKeys.unknownLanguage,
       userL2: l2Code ?? LanguageKeys.unknownLanguage,

lib/pangea/events/event_wrappers/pangea_representation_event.dart

@@ -4,8 +4,9 @@ import 'dart:developer';
 
 import 'package:flutter/foundation.dart';
 
+import 'package:async/async.dart';
 import 'package:collection/collection.dart';
-import 'package:matrix/matrix.dart';
+import 'package:matrix/matrix.dart' hide Result;
 import 'package:matrix/src/utils/markdown.dart';
 import 'package:sentry_flutter/sentry_flutter.dart';
 
@@ -26,6 +27,7 @@ import 'package:fluffychat/pangea/learning_settings/constants/language_constants
 import 'package:fluffychat/pangea/morphs/morph_features_enum.dart';
 import 'package:fluffychat/pangea/morphs/parts_of_speech_enum.dart';
 import 'package:fluffychat/pangea/practice_activities/activity_type_enum.dart';
+import 'package:fluffychat/widgets/future_loading_dialog.dart';
 import 'package:fluffychat/widgets/matrix.dart';
 
 class RepresentationEvent {
@@ -90,11 +92,11 @@ class RepresentationEvent {
     return _tokens?.tokens;
   }
 
-  Future<List<PangeaToken>> tokensGlobal(
+  Future<Result<List<PangeaToken>>> tokensGlobal(
     String senderID,
     DateTime timestamp,
   ) async {
-    if (tokens != null) return tokens!;
+    if (tokens != null) return Result.value(tokens!);
 
     if (_event == null && timestamp.isAfter(DateTime(2024, 9, 25))) {
       Sentry.addBreadcrumb(
@@ -110,8 +112,7 @@ class RepresentationEvent {
         ),
       );
     }
-    final TokensResponseModel res =
-        await MatrixState.pangeaController.messageData.getTokens(
+    final res = await MatrixState.pangeaController.messageData.getTokens(
       repEventId: _event?.eventId,
       room: _event?.room ?? parentMessageEvent.room,
       req: TokensRequestModel(
@@ -128,7 +129,11 @@ class RepresentationEvent {
       ),
     );
 
-    return res.tokens;
+    if (res.isError) {
+      return Result.error(res.error!);
+    } else {
+      return Result.value(res.result!.tokens);
+    }
   }
 
   Future<void> sendTokensEvent(
@@ -146,7 +151,7 @@ class RepresentationEvent {
       return;
     }
 
-    await MatrixState.pangeaController.messageData.sendTokensEvent(
+    await MatrixState.pangeaController.messageData.getTokens(
       repEventId: repEventID,
       room: room,
       req: TokensRequestModel(

lib/pangea/events/repo/tokens_repo.dart

@@ -1,5 +1,6 @@
 import 'dart:convert';
 
+import 'package:async/async.dart';
 import 'package:http/http.dart';
 
 import 'package:fluffychat/pangea/common/config/environment.dart';
@@ -8,39 +9,101 @@ import 'package:fluffychat/pangea/common/network/urls.dart';
 import 'package:fluffychat/pangea/common/utils/error_handler.dart';
 import 'package:fluffychat/pangea/events/repo/token_api_models.dart';
 
+class _TokensCacheItem {
+  final Future<TokensResponseModel> data;
+  final DateTime timestamp;
+
+  const _TokensCacheItem({
+    required this.data,
+    required this.timestamp,
+  });
+}
+
 class TokensRepo {
-  static Future<TokensResponseModel> get(
+  static final Map<String, _TokensCacheItem> _tokensCache = {};
+  static const Duration _cacheDuration = Duration(minutes: 10);
+
+  static Future<Result<TokensResponseModel>> get(
+    String? accessToken,
+    TokensRequestModel request,
+  ) {
+    final cached = _getCached(request);
+    if (cached != null) {
+      return _getResult(request, cached);
+    }
+
+    final future = _fetch(
+      accessToken,
+      request: request,
+    );
+    _setCached(request, future);
+    return _getResult(request, future);
+  }
+
+  static Future<TokensResponseModel> _fetch(
     String? accessToken, {
     required TokensRequestModel request,
   }) async {
     final Requests req = Requests(
-      choreoApiKey: Environment.choreoApiKey,
       accessToken: accessToken,
+      choreoApiKey: Environment.choreoApiKey,
     );
-
     final Response res = await req.post(
       url: PApiUrls.tokenize,
       body: request.toJson(),
     );
 
-    final TokensResponseModel response = TokensResponseModel.fromJson(
-      jsonDecode(
-        utf8.decode(res.bodyBytes).toString(),
-      ),
-    );
-
-    if (response.tokens.isEmpty) {
-      ErrorHandler.logError(
-        e: Exception(
-          "empty tokens in tokenize response return",
-        ),
-        data: {
-          "accessToken": accessToken,
-          "request": request.toJson(),
-        },
+    if (res.statusCode != 200) {
+      throw Exception(
+        'Failed to fetch Tokens data: ${res.statusCode} ${res.reasonPhrase}',
       );
     }
 
-    return response;
+    final Map<String, dynamic> json =
+        jsonDecode(utf8.decode(res.bodyBytes).toString());
+
+    return TokensResponseModel.fromJson(json);
   }
+
+  static Future<Result<TokensResponseModel>> _getResult(
+    TokensRequestModel request,
+    Future<TokensResponseModel> future,
+  ) async {
+    try {
+      final res = await future;
+      return Result.value(res);
+    } catch (e, s) {
+      _tokensCache.remove(request.hashCode.toString());
+      ErrorHandler.logError(
+        e: e,
+        s: s,
+        data: request.toJson(),
+      );
+      return Result.error(e);
+    }
+  }
+
+  static Future<TokensResponseModel>? _getCached(
+    TokensRequestModel request,
+  ) {
+    final cacheKeys = [..._tokensCache.keys];
+    for (final key in cacheKeys) {
+      if (_tokensCache[key]!
+          .timestamp
+          .isBefore(DateTime.now().subtract(_cacheDuration))) {
+        _tokensCache.remove(key);
+      }
+    }
+
+    return _tokensCache[request.hashCode.toString()]?.data;
+  }
+
+  static void _setCached(
+    TokensRequestModel request,
+    Future<TokensResponseModel> response,
+  ) =>
+      _tokensCache[request.hashCode.toString()] = _TokensCacheItem(
+        data: response,
+        timestamp: DateTime.now(),
+      );
 }